mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 08:28:48 +01:00
fix: sanitize nested Git overlays and isolate optional seed verification
Exclude managed repository administrative metadata from raw baselines and overlays, including replay and all-file mode. Keep sanitized Git history transport authoritative. Run optional cache verification only after capacity checks and preserve admission when verification fails. Correct database migration documentation. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
fff60782d2
commit
a44ae3d789
6 files changed
+193
-33
No files matched your search
+1
-1
@@ -739,7 +739,7 @@ transaction boundary.
|
||||
|
||||
### Task workspace records (2026-10-09)
|
||||
|
||||
Migration 0324 makes `execution_workspaces.project_id` nullable and changes its project reference to `ON DELETE RESTRICT`: source-project policy cannot disappear while a workspace still depends on it. `issues.execution_workspace_id` remains the sole active task binding. Versioned selection/pending-selection JSON and a binding revision record intent and protect transitions; they are not additional active bindings.
|
||||
Migration [0331_harsh_newton_destine.sql](../packages/db/src/migrations/0331_harsh_newton_destine.sql) makes `execution_workspaces.project_id` nullable and changes its project reference to `ON DELETE RESTRICT`: source-project policy cannot disappear while a workspace still depends on it. `issues.execution_workspace_id` remains the sole active task binding. Versioned selection/pending-selection JSON and a binding revision record intent and protect transitions; they are not additional active bindings.
|
||||
|
||||
`execution_workspace_repositories` is company-scoped inventory beneath an execution workspace. It records repository identity, contained relative path, requested ref, pinned commit, preparation state and idempotency receipts. Credentials are resolved at use and are never stored in this inventory. Connection/destination `execution_defaults` JSON contains optional typed project/workspace choices, never arbitrary host paths or commands.
|
||||
|
||||
|
||||
@@ -645,6 +645,7 @@ function buildWorkspaceTarExtractCommand(input: {
|
||||
workspaceRemoteDir: string;
|
||||
remoteTar: string;
|
||||
wipeExceptNames: string[] | null;
|
||||
exclude?: string[];
|
||||
}): string {
|
||||
// The wipe must also preserve any in-flight sync scratch tarball at the
|
||||
// workspace root. A concurrent referenced-project upload stages a scratch
|
||||
@@ -661,7 +662,7 @@ function buildWorkspaceTarExtractCommand(input: {
|
||||
: "";
|
||||
return (
|
||||
`mkdir -p ${shellQuote(input.workspaceRemoteDir)}${wipe} && ` +
|
||||
`tar -xf ${shellQuote(input.remoteTar)} -C ${shellQuote(input.workspaceRemoteDir)} && ` +
|
||||
`tar${(input.exclude ?? []).map(entry => ` --exclude=${shellQuote(escapeTarExcludeLiteral(entry))}`).join("")} -xf ${shellQuote(input.remoteTar)} -C ${shellQuote(input.workspaceRemoteDir)} && ` +
|
||||
`rm -f ${shellQuote(input.remoteTar)}`
|
||||
);
|
||||
}
|
||||
@@ -860,9 +861,10 @@ export async function createTarballFromDirectory(input: {
|
||||
async function extractTarballToDirectory(input: {
|
||||
archivePath: string;
|
||||
localDir: string;
|
||||
exclude?: string[];
|
||||
}): Promise<void> {
|
||||
await fs.mkdir(input.localDir, { recursive: true });
|
||||
await execTar(["-xf", input.archivePath, "-C", input.localDir]);
|
||||
await execTar([...(input.exclude ?? []).map(entry => `--exclude=${escapeTarExcludeLiteral(entry)}`), "-xf", input.archivePath, "-C", input.localDir]);
|
||||
}
|
||||
|
||||
async function* walkDirectory(root: string, relative = "", childrenFirst = false): AsyncGenerator<string> {
|
||||
@@ -1207,20 +1209,25 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
// The git enumeration (`git status --ignored`, the HEAD diffs, `ls-files`).
|
||||
// It reads git's own bookkeeping to decide what to include/exclude, so it is
|
||||
// usually fast, but on a large working tree the `--ignored` walk is not free.
|
||||
const gitSnapshot = syncWorkspace && input.workspaceFileMode !== "all"
|
||||
const gitSnapshot = syncWorkspace && (input.workspaceGitSnapshot !== undefined || input.workspaceFileMode !== "all")
|
||||
? input.workspaceGitSnapshot !== undefined
|
||||
? input.workspaceGitSnapshot
|
||||
: await runStepSpan("snapshot.git", () =>
|
||||
readGitWorkspaceSnapshot(input.workspaceLocalDir),
|
||||
)
|
||||
: null;
|
||||
const repositories = gitSnapshot ? (gitSnapshot.repositories ?? []) : input.workspaceRepositories ?? (syncWorkspace && input.workspaceFileMode !== "all" ? await readManagedWorkspaceRepositories(input.workspaceLocalDir) : []);
|
||||
const repositories = gitSnapshot ? (gitSnapshot.repositories ?? []) : input.workspaceRepositories ?? (syncWorkspace ? await readManagedWorkspaceRepositories(input.workspaceLocalDir) : []);
|
||||
const repositoryGitPaths = (children: typeof repositories, prefix = ""): string[] => children.flatMap(repo => {
|
||||
const relative = prefix ? `${prefix}/${repo.path}` : repo.path;
|
||||
return [`${relative}/.git`, ...repositoryGitPaths(repo.snapshot.repositories ?? [], relative)];
|
||||
});
|
||||
const repositoryMetadataExcludes = repositoryGitPaths(repositories);
|
||||
let managedIgnored: Awaited<ReturnType<typeof createWorkspaceManifest>> | null = null;
|
||||
let ignoredPaths = gitSnapshot?.ignoredPaths;
|
||||
let ignoredPaths = input.workspaceFileMode === "all" ? undefined : gitSnapshot?.ignoredPaths;
|
||||
if (!gitSnapshot && repositories.length) {
|
||||
managedIgnored = await createWorkspaceManifest();
|
||||
for (const repo of repositories) {
|
||||
for (const relative of workspacePaths(repo.snapshot.ignoredPaths)) managedIgnored.add("ignored", `${repo.path}/${relative}`);
|
||||
if (input.workspaceFileMode !== "all") for (const relative of workspacePaths(repo.snapshot.ignoredPaths)) managedIgnored.add("ignored", `${repo.path}/${relative}`);
|
||||
for (const relative of workspacePaths(repo.snapshot.deletedPaths)) managedIgnored.add("deleted", `${repo.path}/${relative}`);
|
||||
}
|
||||
ignoredPaths = managedIgnored.paths("ignored");
|
||||
@@ -1230,6 +1237,7 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
let baselineSnapshot: DirectorySnapshot | null = null;
|
||||
const cleanupWorkspaceSnapshot = async () => {
|
||||
await disposeDirectorySnapshot(baselineSnapshot);
|
||||
if (input.workspaceBaseline && input.workspaceBaseline !== baselineSnapshot) await disposeDirectorySnapshot(input.workspaceBaseline);
|
||||
await disposeGitWorkspaceSnapshot(gitSnapshot);
|
||||
if (!gitSnapshot) for (const repo of repositories) await disposeGitWorkspaceSnapshot(repo.snapshot);
|
||||
if (managedIgnored) await fs.rm(path.dirname(managedIgnored.filePath), { recursive: true, force: true });
|
||||
@@ -1247,14 +1255,16 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
const gitIgnoredExcludes = directoryIgnore?.kind === "git" ? directoryIgnore.ignoredPaths : undefined;
|
||||
const workspaceArchiveExclude = mergeExcludes(
|
||||
[".paperclip-runtime"],
|
||||
repositoryMetadataExcludes,
|
||||
input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES,
|
||||
input.workspaceFileMode === "all" ? [] : [...GIT_ARCHIVE_EXCLUDES],
|
||||
input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES],
|
||||
input.workspaceExclude,
|
||||
gitIgnoredExcludes,
|
||||
);
|
||||
const restoreExclude = mergeExcludes(
|
||||
repositoryMetadataExcludes,
|
||||
input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES,
|
||||
input.workspaceFileMode === "all" ? [] : [...GIT_ARCHIVE_EXCLUDES],
|
||||
input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES],
|
||||
[".paperclip-runtime"],
|
||||
input.preserveAbsentOnRestore,
|
||||
input.workspaceExclude,
|
||||
@@ -1275,6 +1285,12 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
}),
|
||||
)))
|
||||
: null;
|
||||
if (input.workspaceBaseline && repositoryMetadataExcludes.length) {
|
||||
baselineSnapshot = await selectDirectorySnapshot(input.workspaceBaseline, {
|
||||
omit: repositoryMetadataExcludes, exclude: mergeExcludes(input.workspaceBaseline.exclude, repositoryMetadataExcludes),
|
||||
ignoredPaths: input.workspaceBaseline.ignoredPaths,
|
||||
});
|
||||
}
|
||||
|
||||
const seedGeneration = input.workspaceSeedCacheDirectory && baselineSnapshot && workspaceInboundMode !== "durable_seed"
|
||||
? workspaceSeedGeneration(baselineSnapshot, gitSnapshot, repositories) : null;
|
||||
@@ -1499,7 +1515,7 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
await copySelectedWorkspaceEntries({
|
||||
sourceRoot: workspaceRoot!,
|
||||
targetDir: workspaceArchiveDir,
|
||||
relativePaths: gitSnapshot?.overlayPaths ?? (baselineSnapshot!.entries instanceof WorkspaceManifestMap ? baselineSnapshot!.entries.manifest : [...baselineSnapshot!.entries.keys()]),
|
||||
relativePaths: (input.workspaceFileMode === "all" ? undefined : gitSnapshot?.overlayPaths) ?? (baselineSnapshot!.entries instanceof WorkspaceManifestMap ? baselineSnapshot!.entries.manifest : [...baselineSnapshot!.entries.keys()]),
|
||||
ignoredPaths,
|
||||
exclude: workspaceArchiveExclude,
|
||||
});
|
||||
@@ -1533,6 +1549,7 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteWorkspaceTar,
|
||||
wipeExceptNames: gitSnapshot || repositories.length ? null : [...preservedNames],
|
||||
exclude: repositoryMetadataExcludes,
|
||||
}),
|
||||
});
|
||||
// 3. Optional remove-deleted-paths command runs LAST, after both extracts.
|
||||
@@ -1553,26 +1570,29 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
// exact seed and verify its selected contents before labeling it a
|
||||
// reusable generation. A mismatch leaves ordinary per-run recovery
|
||||
// intact, but never publishes a mislabeled reusable seed.
|
||||
await runStepSpan("seed.verify", async () => {
|
||||
const verifiedRoot = path.join(tempDir, "verify-seed");
|
||||
if (gitSnapshot || repositories.length) await extractTarballToDirectory({ archivePath: path.join(tempDir, "git-workspace.tar"), localDir: verifiedRoot });
|
||||
await extractTarballToDirectory({ archivePath: path.join(tempDir, "workspace.tar"), localDir: verifiedRoot });
|
||||
for (const relative of workspacePaths(deletedPaths)) {
|
||||
let parent = verifiedRoot;
|
||||
for (const segment of relative.split("/").slice(0, -1)) {
|
||||
parent = path.join(parent, segment);
|
||||
const stat = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; });
|
||||
if (stat && !stat.isDirectory()) return; // Do not follow archive links during cache verification.
|
||||
}
|
||||
await fs.rm(path.join(verifiedRoot, relative), { recursive: true, force: true });
|
||||
}
|
||||
const verified = await captureDirectorySnapshot(verifiedRoot, { exclude: baselineSnapshot!.exclude, ignoredPaths: baselineSnapshot!.ignoredPaths, diskBacked: true });
|
||||
try {
|
||||
if (directorySnapshotSha256(verified) === directorySnapshotSha256(baselineSnapshot!)) {
|
||||
await publishWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, { ...seed, gitArchivePath: gitSnapshot || repositories.length ? seed.gitArchivePath : null }, { companyDirectory: input.workspaceSeedCacheCompanyDirectory });
|
||||
}
|
||||
} finally { await disposeDirectorySnapshot(verified); }
|
||||
});
|
||||
await publishWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration,
|
||||
{ ...seed, gitArchivePath: gitSnapshot || repositories.length ? seed.gitArchivePath : null }, {
|
||||
companyDirectory: input.workspaceSeedCacheCompanyDirectory,
|
||||
verify: () => runStepSpan("seed.verify", async () => {
|
||||
const verifiedRoot = path.join(tempDir, "verify-seed");
|
||||
try {
|
||||
if (gitSnapshot || repositories.length) await extractTarballToDirectory({ archivePath: path.join(tempDir, "git-workspace.tar"), localDir: verifiedRoot });
|
||||
await extractTarballToDirectory({ archivePath: path.join(tempDir, "workspace.tar"), localDir: verifiedRoot, exclude: repositoryMetadataExcludes });
|
||||
for (const relative of workspacePaths(deletedPaths)) {
|
||||
let parent = verifiedRoot;
|
||||
for (const segment of relative.split("/").slice(0, -1)) {
|
||||
parent = path.join(parent, segment);
|
||||
const stat = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; });
|
||||
if (stat && !stat.isDirectory()) return false; // Never follow archive links.
|
||||
}
|
||||
await fs.rm(path.join(verifiedRoot, relative), { recursive: true, force: true });
|
||||
}
|
||||
const verified = await captureDirectorySnapshot(verifiedRoot, { exclude: baselineSnapshot!.exclude, ignoredPaths: baselineSnapshot!.ignoredPaths, diskBacked: true });
|
||||
try { return directorySnapshotSha256(verified) === directorySnapshotSha256(baselineSnapshot!); }
|
||||
finally { await disposeDirectorySnapshot(verified); }
|
||||
} finally { await fs.rm(verifiedRoot, { recursive: true, force: true }); }
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (!stageWorkspace) return;
|
||||
|
||||
@@ -1823,11 +1843,12 @@ export async function prepareSandboxManagedRuntime(input: {
|
||||
workspaceCheckpoint: input.workspaceCheckpoint,
|
||||
onWorkspaceCheckpoint: input.onWorkspaceCheckpoint,
|
||||
workspaceGitSnapshot: repository.snapshot,
|
||||
workspaceFileMode: input.workspaceFileMode,
|
||||
workspaceExclude: nestedExclude,
|
||||
workspaceBaseline: await selectDirectorySnapshot(baselineSnapshot!, {
|
||||
prefix,
|
||||
exclude: mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], nestedExclude),
|
||||
ignoredPaths: repository.snapshot.ignoredPaths,
|
||||
exclude: mergeExcludes(input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], nestedExclude),
|
||||
ignoredPaths: input.workspaceFileMode === "all" ? undefined : repository.snapshot.ignoredPaths,
|
||||
}),
|
||||
onRuntimeProgress: input.onRuntimeProgress,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createTarballFromDirectory, prepareSandboxManagedRuntime, type SandboxSyncOperation } from "./sandbox-managed-runtime.js";
|
||||
import { captureDirectorySnapshot } from "./workspace-restore-merge.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
it.each([undefined, "all"] as const)("keeps nested Git metadata sanitized in cold, cached and recovered seeds (file mode %s)", async (workspaceFileMode) => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), "managed-seed-sanitized-")); roots.push(temp);
|
||||
vi.stubEnv("PAPERCLIP_HOME", path.join(temp, "home"));
|
||||
const host = path.join(temp, "host"), remote = path.join(temp, "remote"), cache = path.join(temp, "cache");
|
||||
const relative = ".paperclip-repositories/fixture";
|
||||
const repository = path.join(host, relative); await fs.mkdir(repository, { recursive: true });
|
||||
const git = async (root: string, ...args: string[]) => (await execute("git", ["-C", root, ...args])).stdout.trim();
|
||||
await git(repository, "init", "-b", "main");
|
||||
await fs.writeFile(path.join(repository, ".gitignore"), "ignored.txt\n");
|
||||
await fs.writeFile(path.join(repository, "work.txt"), "first\n");
|
||||
await git(repository, "add", ".");
|
||||
await git(repository, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-m", "first");
|
||||
await git(repository, "branch", "private-fixture");
|
||||
await fs.writeFile(path.join(repository, "work.txt"), "committed\n");
|
||||
await git(repository, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-am", "selected");
|
||||
const head = await git(repository, "rev-parse", "HEAD");
|
||||
await git(repository, "remote", "add", "origin", "https://fixture-user:fixture-token@example.test/repository.git");
|
||||
await fs.writeFile(path.join(repository, ".git/hooks/post-commit"), "private-hook-fixture");
|
||||
await fs.writeFile(path.join(repository, "work.txt"), "dirty\n");
|
||||
await fs.writeFile(path.join(repository, "ignored.txt"), "ordinary ignored bytes");
|
||||
const run = async (command: string) => { await execute("sh", ["-c", command]); };
|
||||
const sync = async (operations: SandboxSyncOperation[]) => {
|
||||
for (const operation of operations) {
|
||||
for (const file of operation.files) {
|
||||
await fs.mkdir(path.dirname(file.targetPath), { recursive: true });
|
||||
await fs.cp(file.sourcePath, file.targetPath, { recursive: true, verbatimSymlinks: true });
|
||||
// Optional verification scratch must be gone before required upload.
|
||||
if (file.sourcePath.endsWith("/workspace.tar")) {
|
||||
await expect(fs.stat(path.join(path.dirname(file.sourcePath), "verify-seed"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
}
|
||||
}
|
||||
for (const command of operation.postUploadCommands ?? []) await run(command.command);
|
||||
}
|
||||
return { operations: operations.map(operation => ({ operationId: operation.operationId, filesTransferred: 0, bytesTransferred: 0 })) };
|
||||
};
|
||||
const client = { makeDir: (dir: string) => fs.mkdir(dir, { recursive: true }).then(() => {}), run,
|
||||
writeFile: (file: string, data: ArrayBuffer) => fs.writeFile(file, Buffer.from(data)), readFile: (file: string) => fs.readFile(file),
|
||||
listFiles: (dir: string) => fs.readdir(dir), remove: (file: string) => fs.rm(file, { force: true, recursive: true }), syncIn: sync, syncOut: sync };
|
||||
const seed = { workspaceArchivePath: path.join(temp, "run-workspace.tar"), gitArchivePath: path.join(temp, "run-git.tar") };
|
||||
let verifications = 0;
|
||||
const prepare = (overrides: Partial<Parameters<typeof prepareSandboxManagedRuntime>[0]> = {}) => prepareSandboxManagedRuntime({
|
||||
spec: { provider: "test", sandboxId: "test", remoteCwd: remote, timeoutMs: 30000, apiKey: null, transport: "sandbox" },
|
||||
adapterKey: "test", client, workspaceLocalDir: host, workspaceFileMode, workspaceSeedCacheDirectory: cache,
|
||||
workspaceDurableSeed: seed, runtimeSpan: async (name, work) => { if (name === "seed.verify") verifications++; return work(); }, ...overrides,
|
||||
});
|
||||
const assertSanitized = async () => {
|
||||
const checkout = path.join(remote, relative);
|
||||
expect(await git(checkout, "remote", "get-url", "origin")).toBe("https://example.test/repository.git");
|
||||
expect(await fs.readFile(path.join(checkout, ".git/config"), "utf8")).not.toContain("fixture-token");
|
||||
await expect(fs.stat(path.join(checkout, ".git/hooks/post-commit"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(git(checkout, "rev-parse", "--verify", "refs/heads/private-fixture")).rejects.toThrow();
|
||||
expect(await git(checkout, "rev-parse", "HEAD")).toBe(head);
|
||||
expect(await git(checkout, "show", "HEAD:work.txt")).toBe("committed");
|
||||
expect(await fs.readFile(path.join(checkout, "work.txt"), "utf8")).toBe("dirty\n");
|
||||
if (workspaceFileMode === "all") expect(await fs.readFile(path.join(checkout, "ignored.txt"), "utf8")).toBe("ordinary ignored bytes");
|
||||
};
|
||||
const cold = await prepare(); await assertSanitized();
|
||||
expect([...cold.workspaceSyncSnapshot!.baseline.entries].some(([key]) => key.startsWith(`${relative}/.git/`))).toBe(false);
|
||||
expect(await fs.readdir(cache)).toHaveLength(1); expect(verifications).toBe(1);
|
||||
await cold.cleanupWorkspaceSnapshot();
|
||||
await fs.writeFile(path.join(repository, ".git/hooks/post-commit"), "changed private hook");
|
||||
const cached = await prepare(); await assertSanitized(); expect(verifications).toBe(1);
|
||||
// Older descriptors may include raw metadata in their immutable overlay/baseline.
|
||||
const oldBaseline = await captureDirectorySnapshot(host, { diskBacked: true });
|
||||
await createTarballFromDirectory({ localDir: host, archivePath: seed.workspaceArchivePath });
|
||||
const recovered = await prepare({ workspaceInboundMode: "durable_seed", workspaceBaseline: oldBaseline,
|
||||
workspaceGitSnapshot: null, workspaceRepositories: cached.workspaceSyncSnapshot!.repositories });
|
||||
await assertSanitized();
|
||||
expect([...recovered.workspaceSyncSnapshot!.baseline.entries].some(([key]) => key.startsWith(`${relative}/.git/`))).toBe(false);
|
||||
const remoteRepository = path.join(remote, relative);
|
||||
await fs.writeFile(path.join(remoteRepository, "work.txt"), "remote commit\n");
|
||||
await git(remoteRepository, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-am", "remote commit");
|
||||
const remoteHead = await git(remoteRepository, "rev-parse", "HEAD");
|
||||
await fs.writeFile(path.join(remoteRepository, "work.txt"), "restored dirty bytes\n");
|
||||
if (workspaceFileMode === "all") await fs.writeFile(path.join(remoteRepository, "ignored.txt"), "updated ordinary ignored bytes");
|
||||
await recovered.restoreWorkspace();
|
||||
expect(await git(repository, "rev-parse", "HEAD")).toBe(remoteHead);
|
||||
expect(await fs.readFile(path.join(repository, "work.txt"), "utf8")).toBe("restored dirty bytes\n");
|
||||
expect(await fs.readFile(path.join(repository, ".git/config"), "utf8")).toContain("fixture-token");
|
||||
expect(await fs.readFile(path.join(repository, ".git/hooks/post-commit"), "utf8")).toBe("changed private hook");
|
||||
if (workspaceFileMode === "all") expect(await fs.readFile(path.join(repository, "ignored.txt"), "utf8")).toBe("updated ordinary ignored bytes");
|
||||
await recovered.cleanupWorkspaceSnapshot(); await cached.cleanupWorkspaceSnapshot();
|
||||
}, 30000);
|
||||
@@ -121,6 +121,7 @@ it("uses sparse native transfer end-to-end, including replay and unsupported-run
|
||||
let supportsCheckpoint = true;
|
||||
let payloadBytes = -1;
|
||||
let mutateDuringPack = false;
|
||||
let failVerification = false, verificationCalls = 0;
|
||||
const run = async (command: string) => {
|
||||
if (!supportsCheckpoint && command.startsWith("node --input-type=module")) throw new Error("node unavailable");
|
||||
await execute("sh", ["-c", command]);
|
||||
@@ -147,6 +148,12 @@ it("uses sparse native transfer end-to-end, including replay and unsupported-run
|
||||
adapterKey: "test", client, workspaceLocalDir: f.root, workspaceCheckpoint: true,
|
||||
runtimeSpan: async (name, work) => {
|
||||
if (mutateDuringPack && name === "pack") await fs.writeFile(path.join(f.root, "unchanged"), "concurrent change");
|
||||
if (name === "seed.verify") {
|
||||
verificationCalls++;
|
||||
const result = await work();
|
||||
if (failVerification) throw Object.assign(new Error("verification disk full"), { code: "ENOSPC" });
|
||||
return result;
|
||||
}
|
||||
return work();
|
||||
},
|
||||
workspaceSeedCacheDirectory: cache, workspaceDurableSeed: { workspaceArchivePath: path.join(f.temp, "run-seed.tar") }, ...overrides });
|
||||
@@ -176,6 +183,18 @@ it("uses sparse native transfer end-to-end, including replay and unsupported-run
|
||||
expect(await fs.readdir(cache)).toEqual(existingGenerations);
|
||||
expect((await fs.stat(path.join(f.temp, "run-seed.tar"))).size).toBeGreaterThan(0);
|
||||
await raced.cleanupWorkspaceSnapshot();
|
||||
mutateDuringPack = false; failVerification = true;
|
||||
await fs.writeFile(path.join(f.root, "new"), "verification recovery");
|
||||
const verificationFailed = await prepare();
|
||||
expect(await fs.readdir(cache)).toEqual(existingGenerations);
|
||||
const failedSnapshot = verificationFailed.workspaceSyncSnapshot!;
|
||||
await fs.writeFile(path.join(f.root, "new"), "later host bytes");
|
||||
await fs.rm(remote, { recursive: true, force: true });
|
||||
const verificationRecovery = await prepare({ workspaceInboundMode: "durable_seed", workspaceBaseline: failedSnapshot.baseline,
|
||||
workspaceGitSnapshot: failedSnapshot.gitSnapshot, workspaceRepositories: failedSnapshot.repositories });
|
||||
expect(await fs.readFile(path.join(remote, "new"), "utf8")).toBe("verification recovery");
|
||||
await verificationRecovery.cleanupWorkspaceSnapshot();
|
||||
failVerification = false;
|
||||
// A saturated optional cache must still admit and recover from its per-run seed.
|
||||
mutateDuringPack = false;
|
||||
for (let count = (await fs.readdir(cache)).length; count < 16; count++) {
|
||||
@@ -183,7 +202,9 @@ it("uses sparse native transfer end-to-end, including replay and unsupported-run
|
||||
}
|
||||
const saturated = await fs.readdir(cache);
|
||||
await fs.writeFile(path.join(f.root, "new"), "quota recovery");
|
||||
const verificationCallsBeforeSaturation = verificationCalls;
|
||||
const uncached = await prepare();
|
||||
expect(verificationCalls).toBe(verificationCallsBeforeSaturation);
|
||||
expect(await fs.readdir(cache)).toEqual(saturated);
|
||||
const snapshot = uncached.workspaceSyncSnapshot!;
|
||||
await fs.writeFile(path.join(f.root, "new"), "later host edit");
|
||||
|
||||
@@ -94,3 +94,21 @@ it("supports cross-device archive copies independently of the per-run seed", asy
|
||||
const cached = await readWorkspaceSeedGeneration(f.workspace, generation(1));
|
||||
expect(await fs.readFile(cached!.workspaceArchivePath, "utf8")).toBe("run recovery bytes");
|
||||
});
|
||||
|
||||
it("skips optional verification when capacity is exhausted", async () => {
|
||||
const f = await fixture(); const verify = vi.fn(async () => true);
|
||||
expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed, {
|
||||
companyDirectory: f.company, limits: { workspaceGenerations: 0 }, verify,
|
||||
})).toBe(false);
|
||||
expect(verify).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes");
|
||||
});
|
||||
|
||||
it("treats optional verification storage failure as a cache miss", async () => {
|
||||
const f = await fixture();
|
||||
expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed, {
|
||||
companyDirectory: f.company, verify: async () => { throw Object.assign(new Error("disk full"), { code: "ENOSPC" }); },
|
||||
})).toBe(false);
|
||||
expect(await fs.readdir(f.company)).toEqual([]);
|
||||
expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes");
|
||||
});
|
||||
@@ -77,7 +77,7 @@ async function cacheUsage(root: string, limits: SeedCacheLimits): Promise<{ gene
|
||||
* untouched. No eviction: readers may still be using a returned cache path. */
|
||||
export async function publishWorkspaceSeedGeneration(
|
||||
root: string, generation: string, seed: WorkspaceDurableSeedPaths,
|
||||
options: { companyDirectory?: string; limits?: Partial<SeedCacheLimits> } = {},
|
||||
options: { companyDirectory?: string; limits?: Partial<SeedCacheLimits>; verify?: () => Promise<boolean> } = {},
|
||||
): Promise<boolean> {
|
||||
const limits = { ...WORKSPACE_SEED_CACHE_LIMITS, ...options.limits };
|
||||
if (!/^[a-f0-9]{64}$/.test(generation)
|
||||
@@ -119,6 +119,7 @@ export async function publishWorkspaceSeedGeneration(
|
||||
|| workspaces + (workspaceExists ? 0 : 1) > limits.companyWorkspaces) return false;
|
||||
// Reserve enough for a full copy even when hard links are unavailable.
|
||||
assertWorkspaceManifestDiskSpace(company, incomingBytes);
|
||||
if (options.verify && !await options.verify()) return false;
|
||||
await fs.mkdir(root, { recursive: true, mode: 0o700 });
|
||||
const temporary = path.join(root, `.pending-${randomUUID()}`);
|
||||
await fs.mkdir(temporary, { mode: 0o700 });
|
||||
|
||||
Reference in new issue
Block a user