mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 23:20:53 +01:00
perf: reduce repository admission filesystem and write overhead (#16425)
Reduce repository preparation latency by resolving each distinct workspace path once per scan, loading overlapping root provenance, and batching source retention writes. Preserve company authorization, fresh alias checks, and preparation timing. Add a reproducible benchmark and regressions for source identity retention, symlink retargeting, and bounded filesystem and database work. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
5a9e6b6315
commit
69eb3b6eff
8 files changed
+442
-51
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdtemp, realpath, rm } from "node:fs/promises";
|
||||
import fs from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
@@ -113,3 +114,24 @@ it("keeps supplied membership snapshots separate from fresh database snapshots",
|
||||
await db.update(companyMemberships).set({ status: "active" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, userId)));
|
||||
}
|
||||
});
|
||||
|
||||
it("batches source retention and duplicate path reads without losing owners", async () => {
|
||||
const sourceIds = Array.from({ length: 501 }, () => randomUUID());
|
||||
await db.insert(executionWorkspaces).values(sourceIds.map(id => ({ id, companyId, projectId, cwd,
|
||||
sourceIssueId: issueId, name: "Shared source", mode: "shared_workspace", strategyType: "operator_directory" })));
|
||||
const queries: string[] = [];
|
||||
const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: query => queries.push(query) } }) as unknown as typeof db;
|
||||
const paths = vi.spyOn(fs, "realpath");
|
||||
try {
|
||||
await executionWorkspaceRepositoryService(recorded).guardForPublication(input());
|
||||
expect(queries.filter(query => query.startsWith('insert into "execution_workspace_repository_access"')).length).toBeLessThanOrEqual(2);
|
||||
expect(paths.mock.calls.length).toBeLessThanOrEqual(8);
|
||||
} finally {
|
||||
paths.mockRestore();
|
||||
}
|
||||
const retained = await db.select({ scopeId: executionWorkspaceRepositoryAccess.scopeId }).from(executionWorkspaceRepositoryAccess)
|
||||
.where(and(eq(executionWorkspaceRepositoryAccess.companyId, companyId),
|
||||
eq(executionWorkspaceRepositoryAccess.rootKey, createHash("sha256").update(cwd).digest("hex")),
|
||||
eq(executionWorkspaceRepositoryAccess.kind, "source")));
|
||||
expect(new Set(retained.map(row => row.scopeId))).toEqual(new Set([workspaceId, ...sourceIds]));
|
||||
});
|
||||
@@ -691,6 +691,33 @@ exec "$PROBE_REAL_GIT" "$@"
|
||||
await expect(f.service.prepareForAdmission(f.admission)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("rechecks retargeted execution-workspace aliases at admission and publication", async () => {
|
||||
const f = await fixture(), peer = await standardPeer(f);
|
||||
const unrelated = path.join(root, `${f.issueId}-unrelated`); await mkdir(unrelated);
|
||||
const alias = path.join(root, `${f.issueId}-alias`); await symlink(unrelated, alias);
|
||||
await db.update(executionWorkspaces).set({ cwd: alias }).where(eq(executionWorkspaces.id, peer.workspace.id));
|
||||
await db.update(issues).set({ visibility: "private", privacyRootIssueId: peer.issueId, responsibleUserId: peer.userId })
|
||||
.where(eq(issues.id, peer.issueId));
|
||||
await expect(f.service.prepareForAdmission(f.admission)).resolves.toEqual([]);
|
||||
await rm(alias);
|
||||
await symlink(f.admission.cwd, alias);
|
||||
await expect(assertExecutionWorkspaceRootReadAccess(db, { companyId: f.companyId, cwd: f.admission.cwd, actor: f.actor }))
|
||||
.rejects.toThrow();
|
||||
await expect(f.service.prepareForAdmission(f.admission)).rejects.toThrow();
|
||||
await expect(f.service.guardForPublication(f.admission)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.each(["conflicting_path", "overlapping_invalid_key"] as const)("rejects retained root identity corruption (%s)", async scenario => {
|
||||
const f = await fixture();
|
||||
const canonicalCwd = await realpath(f.admission.cwd);
|
||||
const unrelated = path.join(root, `${f.issueId}-unrelated`); await mkdir(unrelated);
|
||||
await db.insert(executionWorkspaceRepositoryAccess).values({ companyId: f.companyId, kind: "source", scopeId: f.workspace.id,
|
||||
rootKey: createHash("sha256").update(scenario === "conflicting_path" ? canonicalCwd : await realpath(unrelated)).digest("hex"),
|
||||
canonicalCwd: scenario === "conflicting_path" ? await realpath(unrelated) : path.join(canonicalCwd, "child") });
|
||||
await expect(f.service.prepareForAdmission(f.admission)).rejects.toThrow("Repository root identity conflicts");
|
||||
await expect(f.service.guardForPublication(f.admission)).rejects.toThrow("Repository root identity conflicts");
|
||||
});
|
||||
|
||||
it.each(["revoked", "deleted_source", "deleted_owner", "retained_reader"] as const)("protects already-copied bytes when post-copy authorization fails (%s)", async failure => {
|
||||
const f = await fixture(), exec = promisify(execFileCallback);
|
||||
await exec("git", ["init", "--initial-branch=main", f.admission.cwd]);
|
||||
|
||||
Reference in new issue
Block a user