mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 16:27:18 +02:00
DottaandPaperclip
d73d7a8651
Batch dependency Markdown reads through the bounded Git snapshot reader and queue newly discovered dependencies once. Preserve package contents, limits, authorization, audits, and reference order. Validation: 63 affected cases; full typecheck/build; exact-head full CI, E2E, Superagent and Apex 5/5. Local Git comparison: 502 individual reads reduced to three batches with identical entries. Co-Authored-By: Paperclip <noreply@paperclip.ing>
237 lines
17 KiB
TypeScript
237 lines
17 KiB
TypeScript
import { execFile } from 'node:child_process';
|
|
import { chmod, mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { promisify } from 'node:util';
|
|
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
|
|
|
const interception = vi.hoisted(() => ({ remote: '', commands: [] as Array<{ args: string[]; cwd: string; env: NodeJS.ProcessEnv }>, hang: false, hangBatch: false, closedFetches: 0, closedBatches: 0 }));
|
|
vi.mock('node:child_process', async importOriginal => {
|
|
const actual = await importOriginal<typeof import('node:child_process')>();
|
|
return { ...actual, spawn: (command: string, args: string[], options: import('node:child_process').SpawnOptions) => {
|
|
interception.commands.push({ args, cwd: String(options.cwd), env: options.env! });
|
|
if (interception.hang && args.includes('fetch')) return actual.spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], options);
|
|
if (interception.hangBatch && args.includes('--batch')) {
|
|
const child = actual.spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], options);
|
|
child.once('close', () => { interception.closedBatches++; });
|
|
return child;
|
|
}
|
|
// Exercise real Git objects/processes without depending on the network in CI.
|
|
const localArgs = args.map(arg => arg === 'https://github.com/acme/skills.git' ? `file://${interception.remote}` : arg);
|
|
if (args.includes('fetch')) localArgs.unshift('-c', 'protocol.file.allow=always');
|
|
const child = actual.spawn(command, localArgs, options);
|
|
if (args.includes('fetch')) child.once('close', () => { interception.closedFetches++; });
|
|
return child;
|
|
} };
|
|
});
|
|
import { clearGitSkillSnapshotCache, openGitSkillSnapshot, parseGitDownloadProgress } from '../services/skill-source-git-snapshot.js';
|
|
import { scanGitHubSkills, scanGitHubSkillsWithPackageFallback, type GitHubRead } from '../services/github-skill-source.js';
|
|
const exec = promisify(execFile);
|
|
let directory: string, commit: string, laterCommit: string;
|
|
const git = (...args: string[]) => exec('git', ['-C', directory, '-c', 'user.name=Test', '-c', 'user.email=test@example.test', ...args]);
|
|
const request = (values: Partial<Parameters<typeof openGitSkillSnapshot>[0]> = {}) => ({ repositoryUrl: 'https://github.com/acme/skills', ref: 'feature/skills', token: 'test-token-never-on-disk', cacheScope: 'company:alice', ...values });
|
|
beforeAll(async () => {
|
|
directory = await mkdtemp(path.join(os.tmpdir(), 'paperclip-snapshot-fixture-'));
|
|
interception.remote = directory;
|
|
await git('init', '-b', 'feature/skills');
|
|
await mkdir(path.join(directory, '.agents/deep/scripts'), { recursive: true });
|
|
await writeFile(path.join(directory, '.agents/deep/SKILL.md'), '---\nname: skill\ndescription: Useful\n---\nInstructions');
|
|
await writeFile(path.join(directory, '.agents/deep/scripts/run.sh'), '#!/bin/sh\ntouch SHOULD_NOT_RUN\n');
|
|
await chmod(path.join(directory, '.agents/deep/scripts/run.sh'), 0o755);
|
|
await writeFile(path.join(directory, '.agents/deep/asset.bin'), Buffer.from([0, 255, 137, 13, 10]));
|
|
await writeFile(path.join(directory, '.gitattributes'), '.agents/deep/asset.bin export-ignore\n*.md export-subst\n');
|
|
await symlink('/etc/passwd', path.join(directory, '.agents/deep/link'));
|
|
await git('add', '.'); await git('commit', '-m', 'Fixture');
|
|
commit = (await git('rev-parse', 'HEAD')).stdout.trim();
|
|
await git('update-index', '--add', '--cacheinfo', `160000,${commit},external/submodule`);
|
|
await git('commit', '-m', 'Gitlink');
|
|
laterCommit = (await git('rev-parse', 'HEAD')).stdout.trim();
|
|
});
|
|
afterEach(async () => { vi.useRealTimers(); interception.hang = false; interception.hangBatch = false; await clearGitSkillSnapshotCache(); interception.commands = []; interception.closedFetches = 0; interception.closedBatches = 0; interception.remote = directory; vi.unstubAllEnvs(); });
|
|
afterAll(async () => { await rm(directory, { recursive: true, force: true }); });
|
|
|
|
describe('Git skill snapshots', () => {
|
|
it('uses real Git batches for transitive dependencies in an oversized repository', async () => {
|
|
const remote = await mkdtemp(path.join(os.tmpdir(), 'paperclip-dependency-blobs-'));
|
|
interception.remote = remote;
|
|
try {
|
|
await exec('git', ['-C', remote, 'init', '-b', 'main']);
|
|
await mkdir(path.join(remote, 'unrelated'));
|
|
await mkdir(path.join(remote, 'shared'));
|
|
const files: Record<string, string> = {
|
|
'shared/guide.md': '[Details](details.md)',
|
|
'shared/details.md': '[Guide](guide.md) [Data](data.json)',
|
|
'shared/data.json': '{"value":true}',
|
|
};
|
|
for (let i = 0; i < 64; i++) {
|
|
await mkdir(path.join(remote, `skills/skill-${i}`), { recursive: true });
|
|
files[`skills/skill-${i}/SKILL.md`] = `---\nname: skill-${i}\ndescription: Dependency fixture\n---\n[Guide](../../shared/guide.md)`;
|
|
}
|
|
for (const [file, content] of Object.entries(files)) await writeFile(path.join(remote, file), content);
|
|
const unrelated = Buffer.alloc(1024 * 1024, 'x');
|
|
for (let i = 0; i < 62; i++) await writeFile(path.join(remote, `unrelated/${i}.bin`), unrelated);
|
|
await exec('git', ['-C', remote, 'add', '.']);
|
|
await exec('git', ['-C', remote, '-c', 'user.name=Test', '-c', 'user.email=test@example.test', 'commit', '-m', 'Dependency fixture']);
|
|
const commitSha = (await exec('git', ['-C', remote, 'rev-parse', 'HEAD'])).stdout.trim();
|
|
const read = (async (url: string) => url.includes('/commits/') ? { sha: commitSha } : { id: 42, full_name: 'acme/skills', default_branch: 'main' }) as GitHubRead;
|
|
read.openSnapshot = (input, options) => openGitSkillSnapshot({ ...request(), ref: input.ref, commitSha: input.commitSha }, options);
|
|
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, read);
|
|
expect(scan.repositorySubset).toBe(true);
|
|
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual(Object.keys(files).sort());
|
|
for (const file of scan.repositoryFiles!) {
|
|
const content = file.encoding === 'base64' ? Buffer.from(file.content, 'base64') : Buffer.from(file.content);
|
|
expect(content.toString()).toBe(files[file.path]);
|
|
}
|
|
const reads = interception.commands.filter(command => command.args.includes('cat-file'));
|
|
expect(reads).toHaveLength(4);
|
|
expect(reads.every(command => command.args.includes('--batch'))).toBe(true);
|
|
} finally {
|
|
await rm(remote, { recursive: true, force: true });
|
|
}
|
|
}, 30_000);
|
|
|
|
it('reads many unrelated support files with one Git batch per scan, including cache hits', async () => {
|
|
const remote = await mkdtemp(path.join(os.tmpdir(), 'paperclip-many-blobs-'));
|
|
interception.remote = remote;
|
|
try {
|
|
await exec('git', ['-C', remote, 'init', '-b', 'main']);
|
|
await mkdir(path.join(remote, 'support'));
|
|
await mkdir(path.join(remote, 'assets'));
|
|
await mkdir(path.join(remote, 'skills/main'), { recursive: true });
|
|
await writeFile(path.join(remote, 'skills/main/SKILL.md'), '---\nname: many-files\ndescription: Shared repository files\n---\nRead the support files.');
|
|
await writeFile(path.join(remote, 'assets/data.bin'), Buffer.from([0, 255, 10]));
|
|
for (let i = 0; i < 1024; i++) await writeFile(path.join(remote, `support/file-${i}.txt`), `Support ${i}\n`);
|
|
await exec('git', ['-C', remote, 'add', '.']);
|
|
await exec('git', ['-C', remote, '-c', 'user.name=Test', '-c', 'user.email=test@example.test', 'commit', '-m', 'Many support files']);
|
|
const commitSha = (await exec('git', ['-C', remote, 'rev-parse', 'HEAD'])).stdout.trim();
|
|
const read = (async (url: string) => url.includes('/commits/') ? { sha: commitSha } : { id: 42, full_name: 'acme/skills', default_branch: 'main' }) as GitHubRead;
|
|
read.openSnapshot = (input, options) => openGitSkillSnapshot({ ...request(), ref: input.ref, commitSha: input.commitSha }, options);
|
|
const first = await scanGitHubSkills({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, read);
|
|
expect(first.repositoryPackage?.error).toBeNull();
|
|
expect(first.repositoryFiles).toHaveLength(1026);
|
|
const second = await scanGitHubSkills({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository', commitSha: first.commitSha }, read);
|
|
expect(second.repositoryFiles).toEqual(first.repositoryFiles);
|
|
expect(interception.commands.filter(command => command.args.includes('fetch'))).toHaveLength(1);
|
|
const reads = interception.commands.filter(command => command.args.includes('cat-file'));
|
|
expect(reads).toHaveLength(2);
|
|
expect(reads.every(command => command.args.includes('--batch'))).toBe(true);
|
|
await expect(readFile(path.join(remote, 'SHOULD_NOT_RUN'))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
} finally {
|
|
await rm(remote, { recursive: true, force: true });
|
|
}
|
|
}, 30_000);
|
|
it('rejects excessive batch file and byte budgets before starting a Git read', async () => {
|
|
const snapshot = await openGitSkillSnapshot(request());
|
|
try {
|
|
const asset = snapshot.entries.find(entry => entry.path.endsWith('asset.bin'))!;
|
|
const before = interception.commands.length;
|
|
await expect(snapshot.readBlobs!(Array(10_000).fill(asset))).rejects.toThrow('Invalid repository file batch');
|
|
await expect(snapshot.readBlobs!(Array(62).fill({ ...asset, size: 1024 * 1024 }))).rejects.toThrow('61 MB');
|
|
expect(interception.commands).toHaveLength(before);
|
|
} finally { await snapshot.release(); }
|
|
});
|
|
it('cancels a blocked Git batch, closes the process, and allows a fresh read', async () => {
|
|
const snapshot = await openGitSkillSnapshot(request());
|
|
try {
|
|
const asset = snapshot.entries.find(entry => entry.path.endsWith('asset.bin'))!;
|
|
interception.hangBatch = true;
|
|
const controller = new AbortController();
|
|
const pending = snapshot.readBlobs!([asset], controller.signal);
|
|
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' });
|
|
await vi.waitFor(() => expect(interception.commands.some(command => command.args.includes('--batch'))).toBe(true));
|
|
controller.abort();
|
|
await rejected;
|
|
expect(interception.closedBatches).toBe(1);
|
|
interception.hangBatch = false;
|
|
expect((await snapshot.readBlobs!([asset])).get(asset.sha)).toEqual(Buffer.from([0, 255, 137, 13, 10]));
|
|
} finally { await snapshot.release(); }
|
|
});
|
|
it('preserves exact bytes, hidden paths and executable bits without checkout, archive exclusions, or submodule traversal', async () => {
|
|
vi.stubEnv('GIT_CONFIG_COUNT', '1'); vi.stubEnv('GIT_CONFIG_KEY_0', 'http.extraHeader'); vi.stubEnv('GIT_CONFIG_VALUE_0', 'DO-NOT-INHERIT');
|
|
vi.stubEnv('GIT_TRACE', '/tmp/do-not-write-git-trace');
|
|
const snapshot = await openGitSkillSnapshot(request());
|
|
expect(snapshot.commitSha).toBe(laterCommit);
|
|
const asset = snapshot.entries.find(entry => entry.path.endsWith('asset.bin'))!;
|
|
expect(await snapshot.readBlob(asset)).toEqual(Buffer.from([0, 255, 137, 13, 10]));
|
|
expect(snapshot.entries.find(entry => entry.path.endsWith('run.sh'))?.mode).toBe('100755');
|
|
expect(snapshot.entries.find(entry => entry.path.endsWith('/link'))?.mode).toBe('120000');
|
|
expect(snapshot.entries.find(entry => entry.path === 'external/submodule')?.type).toBe('commit');
|
|
const fetch = interception.commands.find(command => command.args.includes('fetch'))!;
|
|
expect(fetch.args).toContain('--depth=1');
|
|
expect(fetch.args.join(' ')).not.toContain('test-token');
|
|
expect(fetch.env.PAPERCLIP_GIT_TOKEN).toBe('test-token-never-on-disk');
|
|
const authKey = Object.entries(fetch.env).find(([, value]) => value === 'http.https://github.com/.extraHeader')![0];
|
|
expect(fetch.env[authKey.replace('_KEY_', '_VALUE_')]).toBe(`Authorization: Basic ${Buffer.from('x-access-token:test-token-never-on-disk').toString('base64')}`);
|
|
expect(fetch.env.GIT_TRACE).toBeUndefined();
|
|
expect(Object.values(fetch.env)).not.toContain('DO-NOT-INHERIT');
|
|
expect(await readFile(path.join(fetch.cwd, 'config'), 'utf8')).not.toContain('test-token');
|
|
await expect(readFile(path.join(fetch.cwd, '.agents/deep/SKILL.md'))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
await snapshot.release();
|
|
});
|
|
it('reuses the exact scanned commit for import and isolates snapshots between callers', async () => {
|
|
const beforeDownload = vi.fn();
|
|
const first = await openGitSkillSnapshot(request(), { beforeDownload }); await first.release();
|
|
const pinned = await openGitSkillSnapshot(request({ commitSha: first.commitSha }), { beforeDownload });
|
|
expect(pinned.commitSha).toBe(first.commitSha); await pinned.release();
|
|
expect(interception.commands.filter(command => command.args.includes('fetch'))).toHaveLength(1);
|
|
expect(beforeDownload).toHaveBeenCalledOnce();
|
|
const other = await openGitSkillSnapshot(request({ commitSha: first.commitSha, cacheScope: 'company:bob' })); await other.release();
|
|
expect(interception.commands.filter(command => command.args.includes('fetch'))).toHaveLength(2);
|
|
});
|
|
it('fetches an older pinned commit rather than silently using the current branch', async () => {
|
|
const snapshot = await openGitSkillSnapshot(request({ commitSha: commit }));
|
|
expect(snapshot.commitSha).toBe(commit);
|
|
expect(snapshot.entries.some(entry => entry.type === 'commit')).toBe(false);
|
|
await snapshot.release();
|
|
});
|
|
it('cleans up cancelled downloads, terminates the child and permits retry', async () => {
|
|
interception.hang = true;
|
|
const controller = new AbortController();
|
|
const pending = openGitSkillSnapshot(request(), { signal: controller.signal });
|
|
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' });
|
|
await vi.waitFor(() => expect(interception.commands.some(command => command.args.includes('fetch'))).toBe(true));
|
|
const download = interception.commands.find(command => command.args.includes('fetch'))!;
|
|
controller.abort(); await rejected;
|
|
await expect(readFile(path.join(download.cwd, 'HEAD'))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
interception.hang = false;
|
|
const retry = await openGitSkillSnapshot(request()); expect(retry.commitSha).toBe(laterCommit); await retry.release();
|
|
});
|
|
it.each(['timeout', 'cancel'] as const)('releases both download slots after %s when Git exits but progress never drains', async reason => {
|
|
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
|
|
const controller = new AbortController();
|
|
const onDownload = vi.fn(() => new Promise<void>(() => {}));
|
|
const rejected = [0, 1].map(index => expect(openGitSkillSnapshot(request({ cacheScope: `stalled:${index}` }), {
|
|
signal: controller.signal, onDownload,
|
|
})).rejects.toThrow(reason === 'timeout' ? 'timed out' : 'aborted'));
|
|
await vi.waitFor(() => {
|
|
expect(onDownload).toHaveBeenCalledTimes(2);
|
|
expect(interception.closedFetches).toBe(2);
|
|
});
|
|
const downloads = interception.commands.filter(command => command.args.includes('fetch'));
|
|
await expect(openGitSkillSnapshot(request())).rejects.toThrow('Other repositories are downloading');
|
|
if (reason === 'timeout') await vi.advanceTimersByTimeAsync(180_000);
|
|
else controller.abort();
|
|
await Promise.all(rejected);
|
|
for (const download of downloads) await expect(readFile(path.join(download.cwd, 'HEAD'))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
vi.useRealTimers();
|
|
const retry = await openGitSkillSnapshot(request());
|
|
expect(retry.commitSha).toBe(laterCommit);
|
|
await retry.release();
|
|
});
|
|
it.each(['https://evil.test/acme/skills', 'https://token@github.com/acme/skills', 'file:///tmp/repo'])('rejects unauthorized remote %s before starting Git', async repositoryUrl => {
|
|
await expect(openGitSkillSnapshot(request({ repositoryUrl }))).rejects.toThrow('Invalid GitHub');
|
|
expect(interception.commands).toHaveLength(0);
|
|
});
|
|
it('rejects revision expressions and command options', async () => {
|
|
for (const ref of ['--upload-pack=malicious', 'main:refs/heads/change', 'main~1', 'main^{tree}']) {
|
|
await expect(openGitSkillSnapshot(request({ ref }))).rejects.toThrow();
|
|
}
|
|
expect(interception.commands.some(command => command.args.includes('fetch'))).toBe(false);
|
|
});
|
|
it('reports only numeric download progress, never provider text or credentials', () => {
|
|
expect(parseGitDownloadProgress('Receiving objects: 64% (64/100), 18.50 MiB | 2.0 MiB/s')).toEqual({ stage: 'receiving', percent: 64, receivedBytes: 19_398_656 });
|
|
expect(parseGitDownloadProgress('Resolving deltas: 32% (32/100)')).toEqual({ stage: 'resolving', percent: 32 });
|
|
expect(parseGitDownloadProgress('fatal: token=secret error')).toBeNull();
|
|
});
|
|
});
|