Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-11 21:59:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

perf(skills): batch repository dependency reads (#16377)

Browse Source 
Batch dependency Markdown reads through the bounded Git snapshot reader and queue newly discovered dependencies once. Preserve package contents, limits, authorization, audits, and reference order.

Validation: 63 affected cases; full typecheck/build; exact-head full CI, E2E, Superagent and Apex 5/5. Local Git comparison: 502 individual reads reduced to three batches with identical entries.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-14 22:31:38 -05:00
1 parent 32ede08918
commit d73d7a8651
6 files changed
+120 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/services/github-skill-source.ts
+5 -1
View File
@@ -160,7 +160,11 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
if (dependencyBytes > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
dependencyBlobs.set(entry.sha, bytes);
return bytes;
}) : entries.filter(entry => entry.type !== 'tree');
}, snapshot.readBlobs ? async pending => {
options.signal?.throwIfAborted();
const unread = pending.filter(entry => !dependencyBlobs.has(entry.sha) && (entry.size ?? 0) <= MAX_FILE_BYTES);
if (unread.length) for (const [sha, bytes] of await snapshot.readBlobs!(unread, options.signal)) dependencyBlobs.set(sha, bytes);
} : undefined) : entries.filter(entry => entry.type !== 'tree');
if (repositoryInventory && repositoryInventory.length > MAX_REPOSITORY_FILES) throw unprocessable('Repository package exceeds the 9,990 file limit.');
const readable = repositoryInventory?.filter(entry => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode) && (entry.size ?? 0) <= MAX_FILE_BYTES);
if (readable && readable.reduce((sum, entry) => sum + (entry.size ?? 0), 0) > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
server/src/services/skill-repository-dependencies.ts
+23 -16
View File
@@ -10,10 +10,12 @@ export async function skillRepositoryDependencies(
initial: GitSkillTreeEntry[],
skillDirectories: ReadonlySet<string>,
readBlob: (entry: GitSkillTreeEntry) => Promise<Buffer | null>,
prefetch?: (entries: GitSkillTreeEntry[]) => Promise<void>,
): Promise<GitSkillTreeEntry[]> {
const paths = indexSkillPackagePaths(entries.map(entry => entry.path));
const byPath = new Map(entries.map(entry => [entry.path, entry]));
const selected = new Map<string, GitSkillTreeEntry>();
let pending: GitSkillTreeEntry[] = [];
let bytes = 0;
const add = (entry: GitSkillTreeEntry) => {
if (entry.type === 'tree' || selected.has(entry.path)) return;
@@ -21,26 +23,31 @@ export async function skillRepositoryDependencies(
if (selected.size > MAX_REPOSITORY_FILES) throw unprocessable('Repository package exceeds the 9,990 file limit.');
if (entry.type === 'blob' && ['100644', '100755'].includes(entry.mode) && (entry.size ?? 0) <= 1024 * 1024) bytes += entry.size ?? 0;
if (bytes > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
if (entry.type === 'blob' && ['100644', '100755'].includes(entry.mode) && /\.md$/i.test(entry.path)) pending.push(entry);
};
for (const entry of initial) add(entry);
const manifest = byPath.get('paperclip.skills.json');
if (manifest) add(manifest);
for (const entry of selected.values()) {
if (entry.type !== 'blob' || !['100644', '100755'].includes(entry.mode) || !/\.md$/i.test(entry.path)) continue;
const content = await readBlob(entry);
if (!content) continue;
let root = path.posix.dirname(entry.path);
while (root !== '.' && !skillDirectories.has(root)) root = path.posix.dirname(root);
const file = snapshotFile(path.posix.relative(root, entry.path), 'reference', content);
const inspection = inspectSkillPackage(path.posix.join(root, 'SKILL.md'), [file], paths, {}, [], new Set([entry.path]));
for (const reference of inspection.references) {
const target = reference.resolvedPath;
if (target === '..' || target.startsWith('../') || path.posix.isAbsolute(target)) continue;
const exact = byPath.get(target);
if (exact && exact.type !== 'tree') add(exact);
else if (paths.has(target)) {
const prefix = target === '.' ? '' : `${target}/`;
for (const child of entries) if (child.path.startsWith(prefix)) add(child);
while (pending.length) {
const frontier = pending;
pending = [];
await prefetch?.(frontier);
for (const entry of frontier) {
const content = await readBlob(entry);
if (!content) continue;
let root = path.posix.dirname(entry.path);
while (root !== '.' && !skillDirectories.has(root)) root = path.posix.dirname(root);
const file = snapshotFile(path.posix.relative(root, entry.path), 'reference', content);
const inspection = inspectSkillPackage(path.posix.join(root, 'SKILL.md'), [file], paths, {}, [], new Set([entry.path]));
for (const reference of inspection.references) {
const target = reference.resolvedPath;
if (target === '..' || target.startsWith('../') || path.posix.isAbsolute(target)) continue;
const exact = byPath.get(target);
if (exact && exact.type !== 'tree') add(exact);
else if (paths.has(target)) {
const prefix = target === '.' ? '' : `${target}/`;
for (const child of entries) if (child.path.startsWith(prefix)) add(child);
}
}
}
}