mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-08 12:33:05 +01:00
fix(runtime): bound parallel offline harness backup transfers
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
69dea878c0
commit
919f117092
5 files changed
+225
-49
No files matched your search
@@ -170,3 +170,24 @@ keeps its supervisor and ownership requirements.
|
||||
For a retired reusable sandbox lease, the existing remote-termination resolver
|
||||
may prove destruction through its exact successor handoff chain. A stopped
|
||||
successor, broken chain or competing tip does not grant archival authority.
|
||||
|
||||
### Bounded offline harness transfers
|
||||
|
||||
The full bundled cold rehearsal reached only 699 of 1,953 local harness backups
|
||||
before the 25-minute maintenance deadline. Keep every source and its existing
|
||||
ownership and hash checks. Migrate up to four sources together. Reserve space for
|
||||
both each tar archive and its encrypted copy before starting a group, with the
|
||||
same total scratch limit as one maximum-size serial transfer. Use the verified
|
||||
manifest sizes for a closed directory set; give sources with extra files an
|
||||
exclusive maximum-size reservation so those files are preserved. Enforce the
|
||||
reservation while streaming the archive. Ordinary runtime backups keep their
|
||||
existing maximum archive limit.
|
||||
|
||||
On failure, cancel and await every transfer in the group before returning. A
|
||||
completed publication remains valid for the next attempt. Tests must exercise
|
||||
real PostgreSQL publication and byte restoration, the four-transfer bound,
|
||||
failure while another storage call is unresolved, retry, extra-file preservation,
|
||||
and rejection at the archive limit before any object or row is published. This
|
||||
finite offline operation adds no idle background work. A resumed rehearsal can
|
||||
measure the fix but cannot replace fresh full-image migration and empty-home
|
||||
recovery proof.
|
||||
@@ -25,19 +25,21 @@ const validateLinks = (links: Map<string, string>) => {
|
||||
}
|
||||
}
|
||||
};
|
||||
const boundedArchive = () => {
|
||||
const boundedArchive = (limit = MAX_NATIVE_HARNESS_OBJECT_BYTES) => {
|
||||
let bytes = 0;
|
||||
return new Transform({ transform(chunk, _encoding, callback) {
|
||||
bytes += chunk.length;
|
||||
callback(bytes > MAX_NATIVE_HARNESS_OBJECT_BYTES ? fail() : null, chunk);
|
||||
callback(bytes > limit ? fail() : null, chunk);
|
||||
} });
|
||||
};
|
||||
|
||||
export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal) {
|
||||
export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal, maxArchiveBytes = MAX_NATIVE_HARNESS_OBJECT_BYTES) {
|
||||
if (!Number.isSafeInteger(maxArchiveBytes) || maxArchiveBytes <= 0 || maxArchiveBytes > MAX_NATIVE_HARNESS_OBJECT_BYTES) throw fail();
|
||||
if (!(await lstat(root)).isDirectory()) throw fail();
|
||||
const archive = tar.pack();
|
||||
const output = pipeline(archive, boundedArchive(), createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal });
|
||||
void output.catch(() => undefined);
|
||||
let outputFailure: unknown;
|
||||
const output = pipeline(archive, boundedArchive(maxArchiveBytes), createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal });
|
||||
void output.catch(error => { outputFailure = error; });
|
||||
let count = 0, bytes = 0;
|
||||
const links = new Map<string, string>();
|
||||
const visit = async (relative: string): Promise<void> => {
|
||||
@@ -72,7 +74,11 @@ export async function captureNativeHarnessArchive(root: string, target: string,
|
||||
}
|
||||
};
|
||||
try { await visit(""); validateLinks(links); archive.finalize(); await output; }
|
||||
catch (error) { archive.destroy(error as Error); await output.catch(() => undefined); throw error; }
|
||||
catch (error) {
|
||||
archive.destroy((outputFailure ?? error) as Error);
|
||||
await output.catch(() => undefined);
|
||||
throw outputFailure ?? error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function restoreNativeHarnessArchive(source: string, target: string, signal?: AbortSignal) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { environmentLeases, heartbeatRuns, nativeHarnessBackups, type Db } from
|
||||
import type { StorageService } from "../../storage/types.js";
|
||||
import { nativeHarnessStateBase, readNativeHarnessBackupStamp, verifyNativeHarnessBackupStampAsync } from "./native-harness-backup-stamp.js";
|
||||
import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
|
||||
import { MAX_NATIVE_HARNESS_BYTES, MAX_NATIVE_HARNESS_OBJECT_BYTES } from "./native-harness-backup-archive.js";
|
||||
import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
|
||||
|
||||
export async function hasLocalNativeHarnessObligations(db: Db): Promise<boolean> {
|
||||
@@ -21,6 +22,79 @@ export async function hasLocalNativeHarnessObligations(db: Db): Promise<boolean>
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
const MAX_PARALLEL_BACKUPS = 4;
|
||||
// An archive and its encrypted copy coexist. Keep the same worst-case scratch
|
||||
// bound as one maximum-size serial transfer, including tar header overhead.
|
||||
const MAX_SCRATCH_BYTES = 2 * MAX_NATIVE_HARNESS_OBJECT_BYTES;
|
||||
|
||||
async function prepareSource(base: string, scopeDigest: string) {
|
||||
const root = join(base, scopeDigest, "failover-backups", "current");
|
||||
const stat = await lstat(root).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return null; throw error;
|
||||
});
|
||||
if (!stat) return null;
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("native_harness_migration_unsafe_source");
|
||||
const manifestBytes = await readFile(join(root, "manifest.json"));
|
||||
const manifest = JSON.parse(manifestBytes.toString("utf8")) as Record<string, unknown>;
|
||||
const state = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8"));
|
||||
if (state.lifecycle !== "suspended" || typeof state.runId !== "string"
|
||||
|| manifest.schema !== "paperclip.native-harness-backup.v1") throw new Error("native_harness_migration_source_unsettled");
|
||||
const names = new Set<string>();
|
||||
let bytes = manifestBytes.length, closed = Array.isArray(manifest.directories) && manifest.directories.length > 0;
|
||||
for (const value of Array.isArray(manifest.directories) ? manifest.directories : []) {
|
||||
const entry = value as { name?: unknown; bytes?: unknown } | null;
|
||||
if (!entry || typeof entry.name !== "string" || !/^[A-Za-z0-9._-]+$/.test(entry.name)
|
||||
|| [".", ".."].includes(entry.name) || !Number.isSafeInteger(entry.bytes) || Number(entry.bytes) < 0) { closed = false; break; }
|
||||
names.add(entry.name); bytes += Number(entry.bytes);
|
||||
}
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
closed &&= entries.length === names.size + 1 && entries.every(entry => entry.name === "manifest.json"
|
||||
? entry.isFile() : names.has(entry.name) && entry.isDirectory() && !entry.isSymbolicLink());
|
||||
// Unknown extra files are preserved with an exclusive full-size reservation.
|
||||
const archiveLimit = closed && Number.isSafeInteger(bytes) && bytes <= MAX_NATIVE_HARNESS_BYTES
|
||||
? Math.min(MAX_NATIVE_HARNESS_OBJECT_BYTES, bytes + MAX_NATIVE_HARNESS_OBJECT_BYTES - MAX_NATIVE_HARNESS_BYTES)
|
||||
: MAX_NATIVE_HARNESS_OBJECT_BYTES;
|
||||
return { root, scopeDigest, runId: state.runId as string, archiveLimit,
|
||||
sessionId: manifest.normalizedSessionId, runnerId: manifest.runnerInstanceId,
|
||||
manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}` };
|
||||
}
|
||||
|
||||
type Source = NonNullable<Awaited<ReturnType<typeof prepareSource>>>;
|
||||
|
||||
async function migrateSource(db: Db, input: { storage: StorageService; signal: AbortSignal }, source: Source) {
|
||||
input.signal.throwIfAborted();
|
||||
const { root, scopeDigest, manifestSha256 } = source;
|
||||
const [run] = await db.select({ companyId: heartbeatRuns.companyId, issueId: heartbeatRuns.nativeIssueId,
|
||||
sessionId: heartbeatRuns.nativeSessionId, runnerId: heartbeatRuns.runnerInstanceId }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, source.runId));
|
||||
if (!run) return 0;
|
||||
if (!run.issueId || run.sessionId !== source.sessionId || run.runnerId !== source.runnerId) {
|
||||
throw new Error("native_harness_migration_binding_mismatch");
|
||||
}
|
||||
const [current] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, run.companyId),
|
||||
eq(nativeHarnessBackups.scopeDigest, scopeDigest), eq(nativeHarnessBackups.status, "current")));
|
||||
if (current) {
|
||||
if (current.manifestSha256 !== manifestSha256) throw new Error("native_harness_migration_durable_source_conflict");
|
||||
await verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: run.companyId,
|
||||
scopeDigest, manifestSha256: current.manifestSha256, signal: input.signal });
|
||||
return 1;
|
||||
}
|
||||
const leases = await db.select({ stamp: sql<unknown>`${environmentLeases.metadata}->'nativeHarnessBackup'`,
|
||||
providerLeaseId: environmentLeases.providerLeaseId }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.issueId, run.issueId),
|
||||
sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'manifestSha256' = ${manifestSha256}`));
|
||||
let valid = false;
|
||||
for (const lease of leases) {
|
||||
const stamp = readNativeHarnessBackupStamp(lease.stamp, lease.providerLeaseId ?? "");
|
||||
if (stamp?.sessionScopeSha256 === scopeDigest && await verifyNativeHarnessBackupStampAsync(stamp, lease.providerLeaseId!, input.signal)) { valid = true; break; }
|
||||
}
|
||||
if (!valid) throw new Error("native_harness_migration_backup_unverified");
|
||||
await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest, signal: input.signal,
|
||||
maxArchiveBytes: source.archiveLimit,
|
||||
owner: { kind: "offline_migration", writersStopped: true, companyId: run.companyId, issueId: run.issueId, runId: source.runId } });
|
||||
return 1;
|
||||
}
|
||||
|
||||
export async function migrateNativeHarnessBackups(db: Db, input: {
|
||||
storage: StorageService; writersStopped: true; signal: AbortSignal;
|
||||
}): Promise<number> {
|
||||
@@ -31,49 +105,31 @@ export async function migrateNativeHarnessBackups(db: Db, input: {
|
||||
const directories = await readdir(base).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return [] as string[]; throw error;
|
||||
});
|
||||
let count = 0;
|
||||
let count = 0, reserved = 0, wave: Source[] = [];
|
||||
const flush = async () => {
|
||||
const cancellation = new AbortController();
|
||||
const signal = AbortSignal.any([input.signal, cancellation.signal]);
|
||||
const settled = await Promise.allSettled(wave.map(async source => {
|
||||
try { return await migrateSource(db, { ...input, signal }, source); }
|
||||
catch (error) { cancellation.abort(error); throw error; }
|
||||
}));
|
||||
// Never return failure while a peer can still upload or publish. Partial
|
||||
// completed transfers remain valid for the next offline attempt.
|
||||
if (cancellation.signal.aborted) throw cancellation.signal.reason;
|
||||
for (const result of settled) {
|
||||
if (result.status === "rejected") throw result.reason;
|
||||
count += result.value;
|
||||
}
|
||||
wave = []; reserved = 0;
|
||||
};
|
||||
for (const scopeDigest of directories.filter(name => /^[a-f0-9]{64}$/.test(name)).sort()) {
|
||||
input.signal.throwIfAborted();
|
||||
const root = join(base, scopeDigest, "failover-backups", "current");
|
||||
const stat = await lstat(root).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return null; throw error;
|
||||
});
|
||||
if (!stat) continue;
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("native_harness_migration_unsafe_source");
|
||||
const manifestBytes = await readFile(join(root, "manifest.json"));
|
||||
const manifest = JSON.parse(manifestBytes.toString("utf8"));
|
||||
const state = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8"));
|
||||
if (state.lifecycle !== "suspended" || typeof state.runId !== "string"
|
||||
|| manifest.schema !== "paperclip.native-harness-backup.v1") throw new Error("native_harness_migration_source_unsettled");
|
||||
const [run] = await db.select({ companyId: heartbeatRuns.companyId, issueId: heartbeatRuns.nativeIssueId,
|
||||
sessionId: heartbeatRuns.nativeSessionId, runnerId: heartbeatRuns.runnerInstanceId }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, state.runId));
|
||||
if (!run) continue;
|
||||
if (!run.issueId || run.sessionId !== manifest.normalizedSessionId || run.runnerId !== manifest.runnerInstanceId) {
|
||||
throw new Error("native_harness_migration_binding_mismatch");
|
||||
}
|
||||
const [current] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, run.companyId),
|
||||
eq(nativeHarnessBackups.scopeDigest, scopeDigest), eq(nativeHarnessBackups.status, "current")));
|
||||
const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`;
|
||||
if (current) {
|
||||
if (current.manifestSha256 !== manifestSha256) throw new Error("native_harness_migration_durable_source_conflict");
|
||||
await verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: run.companyId,
|
||||
scopeDigest, manifestSha256: current.manifestSha256, signal: input.signal });
|
||||
count++; continue;
|
||||
}
|
||||
const leases = await db.select({ stamp: sql<unknown>`${environmentLeases.metadata}->'nativeHarnessBackup'`,
|
||||
providerLeaseId: environmentLeases.providerLeaseId }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.issueId, run.issueId),
|
||||
sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'manifestSha256' = ${manifestSha256}`));
|
||||
let valid = false;
|
||||
for (const lease of leases) {
|
||||
const stamp = readNativeHarnessBackupStamp(lease.stamp, lease.providerLeaseId ?? "");
|
||||
if (stamp?.sessionScopeSha256 === scopeDigest && await verifyNativeHarnessBackupStampAsync(stamp, lease.providerLeaseId!, input.signal)) { valid = true; break; }
|
||||
}
|
||||
if (!valid) throw new Error("native_harness_migration_backup_unverified");
|
||||
await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest, signal: input.signal,
|
||||
owner: { kind: "offline_migration", writersStopped: true, companyId: run.companyId, issueId: run.issueId, runId: state.runId } });
|
||||
count++;
|
||||
const source = await prepareSource(base, scopeDigest);
|
||||
if (!source) continue;
|
||||
const reservation = 2 * source.archiveLimit;
|
||||
if (wave.length && (wave.length === MAX_PARALLEL_BACKUPS || reserved + reservation > MAX_SCRATCH_BYTES)) await flush();
|
||||
wave.push(source); reserved += reservation;
|
||||
}
|
||||
if (wave.length) await flush();
|
||||
return count;
|
||||
}
|
||||
@@ -45,6 +45,99 @@ describeEmbeddedPostgres("durable native provider backups", () => {
|
||||
return { ...f, home, root, companyId, owner, scopeDigest, sessionScopeId, storage, identity,
|
||||
manifestSha256: `sha256:${createHash("sha256").update(manifest).digest("hex")}` };
|
||||
}
|
||||
async function offlineFixture(base: string) {
|
||||
const f = await fixture("codex-home"), home = join(base, f.scopeDigest);
|
||||
await rename(f.home, home);
|
||||
const root = join(home, "failover-backups", "current"), runnerInstanceId = randomUUID(), providerLeaseId = randomUUID();
|
||||
const execution = { provider: { kind: "codex" }, binding: { companyId: f.companyId, issueId: f.issueId,
|
||||
runId: f.runId, agentId: environment.current.agentId, executionWorkspaceId: randomUUID() },
|
||||
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: f.identity.normalizedSessionId, driverKind: "codex_app_server",
|
||||
protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } } as never;
|
||||
const manifest = buildNativeHarnessBackupManifest({ backupRoot: root, execution, runnerInstanceId,
|
||||
providerSessionIdentity: { providerSessionId: f.identity.providerSessionId }, sourceProviderLeaseId: providerLeaseId });
|
||||
await writeFile(join(root, "manifest.json"), JSON.stringify(manifest));
|
||||
const stamp = createNativeHarnessBackupStamp({ manifestPath: join(root, "manifest.json"), sessionScopeId: f.sessionScopeId,
|
||||
authorizedProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: f.identity.normalizedSessionId, completedAt: manifest.completedAt });
|
||||
await f.db.update(heartbeatRuns).set({ nativeSessionId: f.identity.normalizedSessionId, runnerInstanceId }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: f.runId,
|
||||
providerLeaseId, status: "expired", metadata: { nativeHarnessBackup: stamp } });
|
||||
return { ...f, home, root };
|
||||
}
|
||||
async function offlineFixtures(count: number) {
|
||||
const base = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(base); await mkdir(base);
|
||||
vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", base);
|
||||
const fixtures = [];
|
||||
for (let i = 0; i < count; i++) fixtures.push(await offlineFixture(base));
|
||||
const runs = await fixtures[0].db.select({ id: heartbeatRuns.id, issueId: heartbeatRuns.nativeIssueId }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, fixtures[0].companyId));
|
||||
for (const run of runs) await settle({ ...fixtures[0], runId: run.id, issueId: run.issueId! }, new Date());
|
||||
return fixtures.sort((a, b) => a.scopeDigest.localeCompare(b.scopeDigest));
|
||||
}
|
||||
it("overlaps verified offline backups within a four-transfer bound and restores their bytes", async () => {
|
||||
const fixtures = await offlineFixtures(6), f = fixtures[0], gate = Promise.withResolvers<void>();
|
||||
const put = f.storage.putFile.bind(f.storage); let active = 0, peak = 0, entered = 0;
|
||||
vi.spyOn(f.storage, "putFile").mockImplementation(async input => {
|
||||
active++; entered++; peak = Math.max(peak, active);
|
||||
try { await gate.promise; return await put(input); } finally { active--; }
|
||||
});
|
||||
const migration = migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(15_000) });
|
||||
try { await vi.waitFor(() => expect(active).toBe(4), { timeout: 3000 }); }
|
||||
finally { gate.resolve(); await migration.catch(() => undefined); }
|
||||
expect(await migration).toBe(6); expect(peak).toBe(4); expect(entered).toBe(6); expect(active).toBe(0);
|
||||
for (const source of fixtures) {
|
||||
await removeNativeHarnessBackup(source.home);
|
||||
const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement);
|
||||
expect(await restoreDurableNativeHarnessBackup({ ...source, storage: f.storage, backupRoot: replacement })).toBe(true);
|
||||
expect(JSON.parse(await readFile(join(replacement, "current", "codex-home", "sessions", "history.json"), "utf8"))).toEqual(source.identity);
|
||||
}
|
||||
});
|
||||
it("waits for cancelled peer transfers before returning failure and permits a complete retry", async () => {
|
||||
const fixtures = await offlineFixtures(2), f = fixtures[0], peerEntered = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const failed = Promise.withResolvers<void>(), put = f.storage.putFile.bind(f.storage); let entered = 0, active = 0, settled = false;
|
||||
const spy = vi.spyOn(f.storage, "putFile").mockImplementation(async input => {
|
||||
const ordinal = ++entered; active++;
|
||||
try {
|
||||
if (ordinal === 1) { await peerEntered.promise; failed.resolve(); throw new Error("storage unavailable"); }
|
||||
peerEntered.resolve(); await release.promise; return await put(input);
|
||||
} finally { active--; }
|
||||
});
|
||||
const migration = migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(15_000) });
|
||||
const completed = migration.then(() => { settled = true; }, () => { settled = true; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(entered).toBe(2), { timeout: 3000 }); await failed.promise;
|
||||
await new Promise<void>(resolve => setImmediate(resolve)); expect(settled).toBe(false);
|
||||
} finally { peerEntered.resolve(); release.resolve(); await completed; }
|
||||
await expect(migration).rejects.toThrow("storage unavailable"); expect(active).toBe(0); spy.mockRestore();
|
||||
for (const source of fixtures) expect(await readFile(join(source.root, "manifest.json"), "utf8")).toContain("paperclip.native-harness-backup.v1");
|
||||
expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(15_000) })).toBe(2);
|
||||
for (const source of fixtures) {
|
||||
const rows = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, source.scopeDigest));
|
||||
expect(rows.filter(row => row.status === "current")).toHaveLength(1);
|
||||
}
|
||||
});
|
||||
it("preserves unlisted files with an exclusive scratch reservation", async () => {
|
||||
const fixtures = await offlineFixtures(3), f = fixtures[0], extra = fixtures[1];
|
||||
await writeFile(join(extra.root, "retained-extra.txt"), "unlisted surviving bytes");
|
||||
const put = f.storage.putFile.bind(f.storage); let active = 0, peak = 0;
|
||||
vi.spyOn(f.storage, "putFile").mockImplementation(async input => {
|
||||
active++; peak = Math.max(peak, active);
|
||||
try { await new Promise<void>(resolve => setImmediate(resolve)); return await put(input); } finally { active--; }
|
||||
});
|
||||
expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(15_000) })).toBe(3);
|
||||
expect(peak).toBe(1);
|
||||
await removeNativeHarnessBackup(extra.home);
|
||||
const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement);
|
||||
expect(await restoreDurableNativeHarnessBackup({ ...extra, storage: f.storage, backupRoot: replacement })).toBe(true);
|
||||
expect(await readFile(join(replacement, "current", "retained-extra.txt"), "utf8")).toBe("unlisted surviving bytes");
|
||||
});
|
||||
it("enforces the reserved archive size while streaming before publishing any object", async () => {
|
||||
const f = await fixture(), put = vi.spyOn(f.storage, "putFile");
|
||||
await expect(persistNativeHarnessBackup({ ...f, maxArchiveBytes: 512 })).rejects.toThrow("native_harness_archive_invalid");
|
||||
expect(put).not.toHaveBeenCalled();
|
||||
expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toHaveLength(0);
|
||||
expect(JSON.parse(await readFile(join(f.root, "codex", "sessions", "history.json"), "utf8"))).toEqual(f.identity);
|
||||
});
|
||||
it.each(["codex", "claude", "opencode", "grok", "new-provider"])("restores %s session bytes after deleting the entire controller home", async provider => {
|
||||
const f = await fixture(provider);
|
||||
await persistNativeHarnessBackup(f);
|
||||
|
||||
@@ -142,7 +142,7 @@ async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner:
|
||||
|
||||
export async function persistNativeHarnessBackup(input: {
|
||||
db: Db; storage: StorageService; owner: NativeRunnerCheckpointFenceOwner | OfflineOwner;
|
||||
scopeDigest: string; root: string; signal?: AbortSignal;
|
||||
scopeDigest: string; root: string; signal?: AbortSignal; maxArchiveBytes?: number;
|
||||
}): Promise<void> {
|
||||
if (!/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail("scope_invalid");
|
||||
const manifestBytes = await readFile(join(input.root, "manifest.json"));
|
||||
@@ -152,7 +152,7 @@ export async function persistNativeHarnessBackup(input: {
|
||||
const key = randomBytes(32);
|
||||
try {
|
||||
const archive = join(scratch, "archive"), encrypted = join(scratch, "encrypted");
|
||||
await captureNativeHarnessArchive(input.root, archive, input.signal);
|
||||
await captureNativeHarnessArchive(input.root, archive, input.signal, input.maxArchiveBytes);
|
||||
const plaintext = await digest(archive, input.signal), iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad(binding));
|
||||
await pipeline(createReadStream(archive), cipher, createWriteStream(encrypted, { flags: "wx", mode: 0o600 }), { signal: input.signal });
|
||||
|
||||
Reference in new issue
Block a user