perf(skills): batch repository dependency reads (#16377)

Batch dependency Markdown reads through the bounded Git snapshot reader and queue newly discovered dependencies once. Preserve package contents, limits, authorization, audits, and reference order.

Validation: 63 affected cases; full typecheck/build; exact-head full CI, E2E, Superagent and Apex 5/5. Local Git comparison: 502 individual reads reduced to three batches with identical entries.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-14 22:31:38 -05:00
1 parent 32ede08918
commit d73d7a8651
6 files changed
+120 -18

No files matched your search

@@ -13,6 +13,28 @@ import { skillFileBytes } from '../services/skill-snapshot.js';
const sha = 'a'.repeat(40);
const md = (name: string) => `---\nname: ${name}\ndescription: A useful skill\n---\nFollow these instructions.\n`;
describe('GitHub skill repository discovery', () => {
it('batches each dependency frontier instead of spawning a Git read for every Markdown file', async () => {
const files = {
...Object.fromEntries(Array.from({ length: 64 }, (_, i) => [`skills/skill-${i}/SKILL.md`, md(`skill-${i}`) + '\n[Guide](../../shared/guide.md)'])),
'shared/guide.md': '[Details](details.md)',
'shared/details.md': '[Guide](guide.md) [Data](data.json)',
'shared/data.json': '{"example":true}',
...Object.fromEntries(Array.from({ length: 10_000 }, (_, i) => [`unrelated/${i}.txt`, ''])),
};
const fixture = githubFixture(files);
const snapshot = await fixture.openSnapshot({ repositoryUrl: '', ref: 'main' });
const read = snapshot.readBlob;
const singleReads = vi.spyOn(snapshot, 'readBlob');
const batches = vi.fn(async (entries: typeof snapshot.entries) => new Map(await Promise.all(entries.map(async entry => [entry.sha, await read(entry)] as const))));
snapshot.readBlobs = batches;
fixture.openSnapshot = async () => snapshot;
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual(Object.keys(files).filter(name => !name.startsWith('unrelated/')).sort());
expect(scan.candidates.every(skill => skill.error === null)).toBe(true);
expect(singleReads).not.toHaveBeenCalled();
expect(batches.mock.calls.map(([entries]) => entries.length)).toEqual([64, 1, 1, 1]);
});
it('keeps skill folders and transitive shared files when unrelated repository files exceed the package limit', async () => {
const fixture = githubFixture({
'skills/main/SKILL.md': md('main') + '\nRead [shared](../../shared/guide.md) and [helper](../helper/SKILL.md).',