diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 5bcf737602d..f75e62d1a09 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -68,6 +68,23 @@ durable checkpoint. Its `original_volume_source_evidence` manifest reason does not grant provider recovery or Runner execution authority. Publication checks the source bytes again and retains the first authenticated archive during a concurrent capture. The receipt binds the exact archive row and checkpoint parts. +Retired authority epochs can use the same evidence archive with a closed +inventory of their original controller and suspended Runner files, bytes and +modes. Each epoch must match a retained same-session run and its canonical +checkpoint identity when a header exists. A genuinely absent historical header +instead requires the exact admitted authority binding and remains explicitly +absent. Malformed, retired, or foreign headers cannot use this path. Current +source checkpoint and artifact requirements remain unchanged. Source fences +recheck the complete inventory before archive and receipt publication. Unknown +children and other nonempty history directories remain blockers. The inventory +and historical headers or their explicit absence stay bound in replacement +validation. History-free manifests and receipts keep their exact +existing shape; current-controller-only evidence remains retained when a later +archive adds historical files. These archives do not become executable backups. +This maintenance inventory is limited to 128 entries, 16 MiB per file and 64 MiB +total. These bounds are intentionally smaller than the Runner control-plane +state's 256 MiB maximum. A larger otherwise valid epoch remains on the original +volume and blocks filesystem independence; it is never truncated or waived. Source locks and a final comparison fence publication; original files, stamps, warnings, failed statuses and session pointers remain unchanged. Startup can validate the published receipt using only database bindings, without the old diff --git a/server/src/services/native-runtime/native-controller-archive.ts b/server/src/services/native-runtime/native-controller-archive.ts index 3a2402ff8ae..b92cad0b6e2 100644 --- a/server/src/services/native-runtime/native-controller-archive.ts +++ b/server/src/services/native-runtime/native-controller-archive.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdtemp, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Db } from "@paperclipai/db"; @@ -8,19 +8,63 @@ import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import type { SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; +export type NativeControllerHistoryEntry = { path: string; mode: number; type: "directory" } + | { path: string; mode: number; type: "file"; content: string }; +export type NativeControllerHistoryManifestEntry = { path: string; mode: number; type: "directory" } + | { path: string; mode: number; type: "file"; size: number; sha256: string }; type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string; + history?: readonly NativeControllerHistoryEntry[]; + historyBindingDigest?: string; purpose?: "filesystem_independence" }; const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); -export function legacyNativeControllerArchive(source: Source) { +export function nativeControllerHistoryManifest(history: readonly NativeControllerHistoryEntry[]): NativeControllerHistoryManifestEntry[] { + return history.map(entry => entry.type === "directory" ? { ...entry } + : { path: entry.path, mode: entry.mode, type: entry.type, size: Buffer.byteLength(entry.content), sha256: sha256(entry.content) }); +} + +export function nativeControllerEvidenceDescriptor(source: { companyId: string; issueId: string; runId: string; + controllerDigest: string; supervisorDigest: string | null; history?: readonly NativeControllerHistoryManifestEntry[]; + historyBindingDigest?: string; + purpose?: "filesystem_independence" }) { + const history = source.history?.length ? source.history.map(entry => entry.type === "directory" + ? { path: entry.path, mode: entry.mode, type: entry.type } + : { path: entry.path, mode: entry.mode, type: entry.type, size: entry.size, sha256: entry.sha256 }) : undefined; + if (history) { + const paths = new Set(); let bytes = 0; + if (source.purpose !== "filesystem_independence" || history.length > 128 || !/^[a-f0-9]{64}$/.test(source.historyBindingDigest ?? "")) { + throw new Error("native_controller_archive_authority_invalid"); + } + for (const entry of history) { + if (!/^authority-epochs(?:\/epoch-[a-f0-9]{24}(?:\/control-plane(?:\/control-plane-state\.json)?|\/runner-state\.json)?)?$/.test(entry.path) + || paths.has(entry.path) || !Number.isInteger(entry.mode) || entry.mode < 0 || (entry.mode & ~0o777) !== 0 + || !["directory", "file"].includes(entry.type) + || entry.path.includes("/") && !paths.has(entry.path.slice(0, entry.path.lastIndexOf("/")))) { + throw new Error("native_controller_archive_authority_invalid"); + } + paths.add(entry.path); + if (entry.type === "file" && (!Number.isSafeInteger(entry.size) || entry.size < 0 || entry.size > 16 * 1024 * 1024 + || !/^[a-f0-9]{64}$/.test(entry.sha256) || (bytes += entry.size) > 64 * 1024 * 1024)) { + throw new Error("native_controller_archive_authority_invalid"); + } + } + } const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId, issueId: source.issueId, runId: source.runId, reason: source.purpose === "filesystem_independence" ? "original_volume_source_evidence" : "original_runner_artifact_unproven", - files: { "control-plane-state.json": sha256(source.controller), - ...(source.supervisor === undefined ? {} : { "supervisor.json": sha256(source.supervisor) }) } }); + files: { "control-plane-state.json": source.controllerDigest, + ...(source.supervisorDigest === null ? {} : { "supervisor.json": source.supervisorDigest }), + ...(history ? Object.fromEntries(history.filter(entry => entry.type === "file").map(entry => [entry.path, entry.sha256])) : {}) }, + ...(history ? { history, historyBindingDigest: source.historyBindingDigest } : {}) }); return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` }; } +export function legacyNativeControllerArchive(source: Source) { + return nativeControllerEvidenceDescriptor({ ...source, controllerDigest: sha256(source.controller), + supervisorDigest: source.supervisor === undefined ? null : sha256(source.supervisor), + history: source.history ? nativeControllerHistoryManifest(source.history) : undefined }); +} + export async function hasArchivedLegacyNativeController(db: Db, input: Source & { storage: StorageService; signal?: AbortSignal }) { return verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: input.companyId, ...legacyNativeControllerArchive(input), signal: input.signal }); @@ -29,7 +73,7 @@ export async function hasArchivedLegacyNativeController(db: Db, input: Source & export async function archiveFailedLegacyNativeController(db: Db, input: Source & { storage: StorageService; writersStopped?: true; onlineOwner?: SettledControllerArchiveOwner; signal: AbortSignal; }) { - if (input.purpose || input.onlineOwner?.filesystemIndependence) throw new Error("native_controller_archive_authority_invalid"); + if (input.purpose || input.history?.length || input.onlineOwner?.filesystemIndependence) throw new Error("native_controller_archive_authority_invalid"); return persistLegacyControllerEvidence(db, input); } @@ -39,6 +83,9 @@ export async function archiveNativeControllerSourceEvidence(db: Db, input: Sourc if (input.purpose !== "filesystem_independence" || !input.onlineOwner.filesystemIndependence) { throw new Error("native_controller_archive_authority_invalid"); } + input = { ...input, history: input.history?.map(entry => ({ ...entry })) }; + (await import("./native-harness-filesystem-independence.js")).assertNativeHarnessFilesystemArchiveContents( + input.onlineOwner, legacyNativeControllerArchive(input)); await db.transaction(async tx => (await import("./native-controller-archive-source.js")) .assertSettledControllerArchiveSource(tx as unknown as Db, input.onlineOwner)); return persistLegacyControllerEvidence(db, input); @@ -60,6 +107,11 @@ async function persistLegacyControllerEvidence(db: Db, input: Source & { await writeFile(join(root, "manifest.json"), descriptor.manifest, { mode: 0o600 }); await writeFile(join(root, "control-plane-state.json"), input.controller, { mode: 0o600 }); if (input.supervisor !== undefined) await writeFile(join(root, "supervisor.json"), input.supervisor, { mode: 0o600 }); + for (const entry of input.history ?? []) { + if (entry.type === "directory") await mkdir(join(root, entry.path), { mode: 0o700 }); + else await writeFile(join(root, entry.path), entry.content, { mode: entry.mode }); + } + for (const entry of [...(input.history ?? [])].reverse()) await chmod(join(root, entry.path), entry.mode); await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest: descriptor.scopeDigest, signal: input.signal, owner: input.onlineOwner ?? { kind: "offline_migration", writersStopped: true, terminalFailureOnly: true, companyId: input.companyId, issueId: input.issueId, runId: input.runId } }); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts index 0d0793a0eec..38a99f5c080 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, truncate, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { and, eq } from "drizzle-orm"; @@ -30,6 +30,9 @@ import { createNativeControllerRecovery } from "./native-controller-recovery.js" import { ensureNativeCompletionContract } from "./completion-contracts.js"; import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { nativeCheckpointRetirement } from "./native-runner-checkpoint-retirement.js"; +import { nativeSha256 } from "./canonical.js"; +import { nativeControllerEvidenceDescriptor } from "./native-controller-archive.js"; vi.mock("../../build-commit.js", () => ({ readBuildCommit: () => "a".repeat(40) })); @@ -40,7 +43,7 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await db.delete(agentTaskSessions); for (const root of roots.splice(0)) await removeRuntimeAssetTree(root); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); - async function fixture(kind: "unversioned" | "repository" | "package" | "none" = "unversioned") { + async function fixture(kind: "unversioned" | "repository" | "package" | "none" = "unversioned", managedWorkspace = false) { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); const mount = await realpath(await mkdtemp(join(tmpdir(), "native-harness-original-"))); roots.push(mount); const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); @@ -88,7 +91,7 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext); } const execution = parseNativeExecutionInput({ schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, - binding: { companyId, agentId, issueId, runId, executionWorkspaceId: runId }, provider: { kind: "codex", model: null }, + binding: { companyId, agentId, issueId, runId, executionWorkspaceId: managedWorkspace ? randomUUID() : runId }, provider: { kind: "codex", model: null }, task: { identifier: "FS-1", title: "Retain failure", description: null, prompt: "Retain failure", workMode: "standard" }, workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId: sessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, @@ -122,8 +125,8 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" return { input, runId, issueId, agentId, leaseId, execution, witness, storage, mount, sourceRoot: scopedRunnerdStateRoot(execution) }; } - async function controllerFixture() { - const f = await fixture(); + async function controllerFixture(managedWorkspace = false) { + const f = await fixture("unversioned", managedWorkspace); const completion = await ensureNativeCompletionContract({ db, companyId: f.input.companyId, issue: { id: f.issueId, title: "Retain controller evidence", description: null }, actorId: f.agentId }); const execution = parseNativeExecutionInput({ ...f.execution, session: { ...f.execution.session, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } }, @@ -168,9 +171,256 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" { runId: f.runId, runnerInstanceId: prior!.runnerInstanceId!, normalizedSessionId: execution.session.normalizedSessionId! }) } }).where(eq(heartbeatRuns.id, f.runId)); await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result!.id, recoveryState: null, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, f.runId)); - return { ...f, execution, controllerPath, raw, supervisor, supervisorPath: descriptor.supervisorPath }; + return { ...f, execution, controllerPath, raw, supervisor, supervisorPath: descriptor.supervisorPath, identity }; } + async function historyFixture(keepHistoricalHeaders = false, includeRunnerState = false) { + const f = await controllerFixture(true); + const get = f.storage.getObject.bind(f.storage); + const held = vi.spyOn(f.storage, "getObject").mockImplementation(async (...args) => { + if (args[1].includes("/skill-packages/")) throw new Error("fixture package verification held"); + return get(...args); + }); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("fixture package verification held"); + held.mockRestore(); + const [currentOnly] = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.runId, f.runId)); + expect(currentOnly?.status).toBe("current"); + const [anchor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const entries: Array<{ path: string; contents: string; runId: string; header: typeof nativeRunnerCheckpoints.$inferSelect; + runner?: { path: string; contents: string } }> = []; + for (let index = 1; index <= 3; index++) { + const runId = randomUUID(), bootId = randomUUID(), leaseOwner = randomUUID(), leaseId = randomUUID(); + const execution = parseNativeExecutionInput({ ...f.execution, binding: { ...f.execution.binding, runId } }); + const snapshot = { ...(anchor!.runnerProfileJson!.sessionCheckpoint as Record), + identity: { companyId: f.input.companyId, agentId: f.agentId, issueId: f.issueId, runId, + sessionId: execution.session.normalizedSessionId } }; + await db.insert(heartbeatRuns).values({ ...anchor!, id: runId, status: "running", finishedAt: null, + createdAt: new Date(anchor!.createdAt.getTime() - index * 60_000), + runnerProfileJson: { ...anchor!.runnerProfileJson, nativeExecutionInput: execution, sessionCheckpoint: snapshot } }); + await db.insert(nativeRunFinalizations).values({ companyId: f.input.companyId, issueId: f.issueId, runId, + phase: "observed", controllerBootId: bootId, controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) }); + const identity = { ...f.identity, environmentLeaseId: execution.binding.executionWorkspaceId, runId, turnId: randomUUID(), itemId: randomUUID() }; + let epoch = "epoch-" + createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 24); + let path = join("authority-epochs", epoch, "control-plane/control-plane-state.json"); + const control = new DurablePrpControlPlane({ stateDirectory: join(f.sourceRoot, "authority-epochs", epoch, "control-plane"), + identity, expectedRunnerVersion: "raw-evidence-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + for (const type of ["run.prepare", "session.open", "turn.start", "runner.suspend"]) { + Object.assign(control.queueCommand(type), { status: "completed", result: { status: "completed" } }); + } + control.store.state.committedEvents.push({ sourceSeq: 1, sourceEventId: `retired-stream-${index}`, eventType: "item.delta", priority: 1, + envelope: { ...identity, text: `Retained historical stream ${index}` }, deliveryCount: 1, logicalEffectCount: 1 }); + control.store.state.ackedSourceSeq = 1; + const contents = JSON.stringify(control.store.state); + control.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, { companyId: f.input.companyId, issueId: f.issueId, + runId, authorityRunId: runId, bootId, generation: 1, leaseOwner })); + await control.flushCheckpoint(); await control.stop(); + const canonicalEpoch = "epoch-" + createHash("sha256").update(JSON.stringify(JSON.parse(contents).identity)).digest("hex").slice(0, 24); + if (canonicalEpoch !== epoch) { + await rename(join(f.sourceRoot, "authority-epochs", epoch), join(f.sourceRoot, "authority-epochs", canonicalEpoch)); + epoch = canonicalEpoch; path = join("authority-epochs", epoch, "control-plane/control-plane-state.json"); + } + await writeFile(join(f.sourceRoot, path), contents, { mode: 0o600 }); + const runner = includeRunnerState ? { path: join("authority-epochs", epoch, "runner-state.json"), contents: JSON.stringify({ + schema: "paperclip.runner.durable.state.v1", ...JSON.parse(contents).identity, lifecycle: "suspended", + nextSourceSeq: 2, ackedSourceSeq: 1, lastControllerCommandSeq: 4, reconnectCount: 0, + outbox: [{ sourceSeq: 2, sourceEventId: `retired-outbox-${index}`, eventType: "item.delta", + payload: { text: `Suspended historical outbox ${index}` } }], processedCommands: {}, diagnostics: ["retained suspended authority"], + }) } : undefined; + if (runner) await writeFile(join(f.sourceRoot, runner.path), runner.contents, { mode: 0o600 }); + const [result] = await db.insert(nativeRunResults).values({ companyId: f.input.companyId, issueId: f.issueId, runId, + completionContractId: anchor!.completionContractId!, serverFingerprint: "historical-evidence-fixture", schemaStatus: "accepted", + resultJson: {}, canonicalSha256: "a".repeat(64) }).returning(); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result!.id, leaseOwner: null, leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, runId)); + const [header] = await db.select().from(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.runId, runId), eq(nativeRunnerCheckpoints.part, "header"))); + expect(header).toBeDefined(); + if (!keepHistoricalHeaders) await db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, runId)); + const providerLeaseId = `destroyed-${randomUUID()}`; + await db.insert(environmentLeases).values({ ...lease!, id: leaseId, heartbeatRunId: runId, providerLeaseId, + metadata: { ...lease!.metadata, nativeHarnessBackup: { ...(lease!.metadata!.nativeHarnessBackup as Record), + sourceProviderLeaseId: providerLeaseId, authorizedProviderLeaseId: providerLeaseId }, + remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: f.input.companyId, runId, leaseId, + provider: "daytona", providerLeaseId, state: "destroyed", confirmedAt: new Date().toISOString() } } }); + entries.push({ path, contents, runId, header: header!, runner }); + } + return { ...f, entries, currentOnly: currentOnly! }; + } + + it.each([false, true])("preserves a four-run session's three authority epochs with suspended Runner state %s and validates without the original volume", async includeRunnerState => { + const f = await historyFixture(false, includeRunnerState); + const beforeRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId)); + const beforeLeases = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId)); + for (const entry of f.entries) expect(await db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, entry.runId))).toEqual([]); + const result = await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + const [anchor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = anchor!.runnerProfileJson!.nativeHarnessFilesystemIndependence as { controller: { archive: { id: string } } }; + const [archive] = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, receipt.controller.archive.id)); + expect(archive!.id).not.toBe(f.currentOnly.id); + expect((await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, f.currentOnly.id)))[0]).toEqual(f.currentOnly); + const target = join(f.mount, "isolated-authority-evidence"); + expect(await restoreDurableNativeHarnessBackup({ db, storage: f.storage, companyId: f.input.companyId, + scopeDigest: archive!.scopeDigest, backupRoot: target })).toBe(true); + for (const entry of f.entries) { + for (const file of [entry, ...(entry.runner ? [entry.runner] : [])]) { + expect(await readFile(join(target, "current", file.path), "utf8")).toBe(file.contents); + expect((await lstat(join(target, "current", file.path))).mode & 0o777).toBe(0o600); + expect(await readFile(join(f.sourceRoot, file.path), "utf8")).toBe(file.contents); + } + } + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId))).sort((a, b) => a.id.localeCompare(b.id))) + .toEqual(beforeRuns.map(run => run.id === f.runId ? { ...run, runnerProfileJson: { ...run.runnerProfileJson, + nativeHarnessFilesystemIndependence: anchor!.runnerProfileJson!.nativeHarnessFilesystemIndependence } } : run).sort((a, b) => a.id.localeCompare(b.id))); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId))).toEqual(beforeLeases); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-replacement")); vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-replacement/state")); + vi.stubEnv("RAILWAY_VOLUME_ID", ""); vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("replacement must not read storage")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(result.receiptDigest).toMatch(/^[a-f0-9]{64}$/); + }); + + it.each(["active", "foreign-identity"])("retains historical Runner state without publication for %s", async control => { + const f = await historyFixture(false, true), runner = f.entries[0]!.runner!, value = JSON.parse(runner.contents); + if (control === "active") value.lifecycle = "active"; + else value.runId = randomUUID(); + const contents = JSON.stringify(value), path = join(f.sourceRoot, runner.path); + await writeFile(path, contents); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("native_harness_filesystem_independence_unverified"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect((await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, f.currentOnly.id)))[0]).toEqual(f.currentOnly); + expect(await readFile(path, "utf8")).toBe(contents); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + }); + + it("bounds historical evidence at sixteen MiB per file and sixty-four MiB total", () => { + const limit = 16 * 1024 * 1024; + const descriptor = (sizes: number[]) => nativeControllerEvidenceDescriptor({ companyId: randomUUID(), issueId: randomUUID(), + runId: randomUUID(), controllerDigest: "a".repeat(64), supervisorDigest: null, purpose: "filesystem_independence", + historyBindingDigest: "b".repeat(64), history: [{ path: "authority-epochs", mode: 0o700, type: "directory" }, + ...sizes.flatMap((size, index) => { + const root = `authority-epochs/epoch-${index.toString(16).padStart(24, "0")}`; + return [{ path: root, mode: 0o700, type: "directory" as const }, + { path: `${root}/control-plane`, mode: 0o700, type: "directory" as const }, + { path: `${root}/control-plane/control-plane-state.json`, mode: 0o600, type: "file" as const, size, sha256: "c".repeat(64) }]; + })] }); + expect(() => descriptor([limit])).not.toThrow(); + expect(() => descriptor([limit + 1])).toThrow("native_controller_archive_authority_invalid"); + expect(() => descriptor([limit, limit, limit, limit])).not.toThrow(); + expect(() => descriptor([limit, limit, limit, limit, 1])).toThrow("native_controller_archive_authority_invalid"); + }); + + it("retains an oversized historical file and its existing archive without publication", async () => { + const f = await historyFixture(), path = join(f.sourceRoot, f.entries[0]!.path); + await truncate(path, 20 * 1024 * 1024); + const put = vi.spyOn(f.storage, "putFile"); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("native_harness_filesystem_independence_unverified"); + expect(put).not.toHaveBeenCalled(); + expect((await lstat(path)).size).toBe(20 * 1024 * 1024); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect((await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, f.currentOnly.id)))[0]).toEqual(f.currentOnly); + }); + + it.each(["foreign", "unknown", "active-owner", "changed-after-upload", "added-after-upload"])("retains authority history without publication for %s", async control => { + const f = await historyFixture(), path = join(f.sourceRoot, f.entries[0]!.path); + if (control === "foreign") { + const value = JSON.parse(f.entries[0]!.contents); value.identity.runId = randomUUID(); + await writeFile(path, JSON.stringify(value)); + } + if (control === "unknown") await writeFile(join(path, "..", "unknown.json"), "uncovered retained data"); + if (control === "active-owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "still-owned", leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, f.entries[0]!.runId)); + if (control.endsWith("after-upload")) { + const put = f.storage.putFile.bind(f.storage); + vi.spyOn(f.storage, "putFile").mockImplementation(async args => { + const result = await put(args); + if (control === "changed-after-upload") await writeFile(path, f.entries[0]!.contents + "\n"); + else await writeFile(join(path, "..", "new-uncovered.json"), "new retained data"); + return result; + }); + } + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect((await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, f.currentOnly.id)))[0]).toEqual(f.currentOnly); + expect((await lstat(path)).isFile()).toBe(true); + }); + + it("invalidates authority history coverage when a retained historical header disappears", async () => { + const f = await historyFixture(true); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + await db.delete(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.runId, f.entries[0]!.runId), eq(nativeRunnerCheckpoints.part, "header"))); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-replacement")); vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-replacement/state")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + expect(await readFile(join(f.sourceRoot, f.entries[0]!.path), "utf8")).toBe(f.entries[0]!.contents); + }); + + it.each(["new-header", "receipt-identity", "missing-history"])("invalidates published authority history after %s changes", async control => { + const f = await historyFixture(); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + if (control === "new-header") await db.insert(nativeRunnerCheckpoints).values(f.entries[0]!.header); + else { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const profile = structuredClone(run!.runnerProfileJson!); + const receipt = profile.nativeHarnessFilesystemIndependence as { controller: { history?: { epochs: Array<{ identity: { turnId: string }; identityDigest: string }> } } }; + if (control === "missing-history") delete receipt.controller.history; + else { + receipt.controller.history!.epochs[0]!.identity.turnId = randomUUID(); + receipt.controller.history!.epochs[0]!.identityDigest = nativeSha256(receipt.controller.history!.epochs[0]!.identity); + } + await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId)); + } + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-replacement")); vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-replacement/state")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + expect(await readFile(join(f.sourceRoot, f.entries[0]!.path), "utf8")).toBe(f.entries[0]!.contents); + }); + + it.each(["malformed", "retired", "unknown-override"])("refuses historical authority coverage with %s identity evidence", async control => { + const f = await historyFixture(control !== "unknown-override"), entry = f.entries[0]!; + if (control === "malformed") await db.update(nativeRunnerCheckpoints).set({ material: { schema: "unreadable" } }) + .where(and(eq(nativeRunnerCheckpoints.runId, entry.runId), eq(nativeRunnerCheckpoints.part, "header"))); + if (control === "retired") await db.update(nativeRunnerCheckpoints).set({ material: nativeCheckpointRetirement({ companyId: f.input.companyId, + agentId: f.agentId, sessionId: f.execution.session.normalizedSessionId!, runIds: [entry.runId] }) }) + .where(and(eq(nativeRunnerCheckpoints.runId, entry.runId), eq(nativeRunnerCheckpoints.part, "header"))); + if (control === "unknown-override") { + const value = JSON.parse(entry.contents); value.identity.environmentLeaseId = randomUUID(); + await writeFile(join(f.sourceRoot, entry.path), JSON.stringify(value)); + const previous = entry.path.split("/")[1]!, next = "epoch-" + createHash("sha256").update(JSON.stringify(value.identity)).digest("hex").slice(0, 24); + await rename(join(f.sourceRoot, "authority-epochs", previous), join(f.sourceRoot, "authority-epochs", next)); + } + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect((await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, f.currentOnly.id)))[0]).toEqual(f.currentOnly); + }); + + it("retains the first complete authority-history archive and receipt when an upload finishes late", async () => { + const f = await historyFixture(); + let release!: () => void, entered!: () => void, delay = true; + const blocked = new Promise(resolve => { release = resolve; }), ready = new Promise(resolve => { entered = resolve; }); + const put = f.storage.putFile.bind(f.storage); + vi.spyOn(f.storage, "putFile").mockImplementation(async args => { + if (delay) { delay = false; entered(); await blocked; } + return put(args); + }); + const late = preserveNativeHarnessFilesystemIndependence(db, f.input); + void late.catch(() => undefined); + try { + await ready; + const winner = await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + release(); expect((await late).receiptDigest).toBe(winner.receiptDigest); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(published); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + } finally { release(); await late.catch(() => undefined); } + }); + it("publishes only independence and validates on an empty replacement without the source capability", async () => { const f = await fixture(); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); @@ -311,9 +561,14 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" expect(await readFile(join(restored, "current/supervisor.json"), "utf8")).toBe(f.supervisor); expect(JSON.parse(await readFile(join(restored, "current/manifest.json"), "utf8"))) .toMatchObject({ schema: "paperclip.legacy-controller-archive.v1", reason: "original_volume_source_evidence" }); + expect(await readFile(join(restored, "current/manifest.json"), "utf8")).toBe(JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", + companyId: f.input.companyId, issueId: f.issueId, runId: f.runId, reason: "original_volume_source_evidence", + files: { "control-plane-state.json": createHash("sha256").update(f.raw).digest("hex"), + "supervisor.json": createHash("sha256").update(f.supervisor).digest("hex") } })); expect(await readFile(f.controllerPath, "utf8")).toBe(f.raw); expect(await readFile(f.supervisorPath, "utf8")).toBe(f.supervisor); const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect((after!.runnerProfileJson!.nativeHarnessFilesystemIndependence as { controller: unknown }).controller).not.toHaveProperty("history"); expect(after).toEqual({ ...before, runnerProfileJson: { ...before!.runnerProfileJson, nativeHarnessFilesystemIndependence: after!.runnerProfileJson!.nativeHarnessFilesystemIndependence } }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(owner); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.ts index e2c2eff9284..ebed5d1f0e8 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.ts @@ -1,5 +1,5 @@ import { constants } from "node:fs"; -import { lstat, open, readdir } from "node:fs/promises"; +import { lstat, open, opendir, readdir, realpath } from "node:fs/promises"; import { createHash } from "node:crypto"; import { basename, join, posix } from "node:path"; import { and, asc, eq, inArray, or, sql } from "drizzle-orm"; @@ -14,7 +14,8 @@ import { skillPackageStore } from "../skill-package-store.js"; import { skillPackageManifest } from "../skill-package-files.js"; import { assertSkillSnapshotPath, skillFileBytes } from "../skill-snapshot.js"; import { REPOSITORY_CONTENT_DIR, repositorySkillProjection } from "../skill-repository-package.js"; -import { captureLegacyDurablePrpCheckpoint, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, parseNativeExecutionInput, + type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { nativeSha256 } from "./canonical.js"; import { readRuntimeSkillPreservation } from "./runtime-skill-preservation-reference.js"; import { findNativeSessionResumeRun } from "./native-session-resume.js"; @@ -24,9 +25,10 @@ import { tryLockNativeWorkspaceFinalization } from "./native-workspace-finalizat import { assertOriginalHarnessVolumeSource, type OriginalHarnessVolumeCapability } from "./native-harness-original-volume.js"; import { isKnownNativeSessionRootEntry } from "./native-session-root-entries.js"; import { readLegacyNativeControllerArchiveSource, readLegacyNativeControllerArchiveDigests } from "./native-controller-migration.js"; -import { archiveNativeControllerSourceEvidence, legacyNativeControllerArchive } from "./native-controller-archive.js"; +import { archiveNativeControllerSourceEvidence, legacyNativeControllerArchive, nativeControllerEvidenceDescriptor, + nativeControllerHistoryManifest, type NativeControllerHistoryEntry, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; import { nativeControllerArchiveCoordinatorDigest, nativeControllerArchiveProfileDigest, type SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; -import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { loadNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; import { readNativeControllerRecovery } from "./native-controller-recovery.js"; import { verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; @@ -39,14 +41,18 @@ function fail(): never { throw new Error("native_harness_filesystem_independence const profile = (value: Record | null) => { const copy = { ...value }; delete copy[KEY]; return copy; }; type PackageBinding = { id: string; digest: string }; type VersionBinding = { id: string; digest: string }; -type ControllerBinding = { checkpointPartsDigest: string; archive: null | { id: string; digest: string; +type HistoryEpoch = { name: string; runId: string; identity: DurableRecoveryIdentity; identityDigest: string; headerDigest: string | null }; +type HistoryBinding = { entries: NativeControllerHistoryManifestEntry[]; epochs: HistoryEpoch[] }; +type CapturedHistory = { entries: NativeControllerHistoryEntry[]; binding: HistoryBinding; localDigest: string }; +type ControllerBinding = { checkpointPartsDigest: string; history?: HistoryBinding; archive: null | { id: string; digest: string; controllerDigest: string; supervisorDigest: string | null } }; type Receipt = { schema: typeof SCHEMA; selectedSessionId: string; lastRunId: string; sourceRunId: string; scopeDigest: string; factsDigest: string; sourceCapabilityFingerprint: string; localFactsDigest: string; assetsDigest: string; packages: PackageBinding[]; versions: VersionBinding[]; controller: ControllerBinding; observedAt: string }; type Scope = { companyId: string; selectedSessionId: string; scopeDigest: string }; type Run = typeof heartbeatRuns.$inferSelect; -const evidenceOwners = new WeakMap(); +const evidenceOwners = new WeakMap(); async function databaseFacts(db: Db, input: Scope, lock: boolean) { const [selection] = await db.select().from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, input.companyId), @@ -120,7 +126,7 @@ async function databaseFacts(db: Db, input: Scope, lock: boolean) { const factsDigest = digest({ selection, runs: runs.map(run => ({ ...run, runnerProfileJson: profile(run.runnerProfileJson) })), owners, task: { id: task.id, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId }, leases, terminations, sourceRunId: source.id, sourceProfile: source.runnerProfileJson }); - return { anchor, execution, source, factsDigest }; + return { anchor, execution, source, factsDigest, runs }; } async function assetBindings(db: Db, anchor: Run, sourceRunId: string) { @@ -179,7 +185,7 @@ export async function hasValidNativeHarnessFilesystemIndependence(db: Db, input: const facts = await databaseFacts(db, input, false); const assets = await assetBindings(db, facts.anchor, facts.source.id); const existing = facts.anchor.runnerProfileJson?.[KEY] as Receipt | undefined; - const controller = await controllerBinding(db, facts, existing?.controller?.archive ?? null, false); + const controller = await controllerBinding(db, facts, existing?.controller?.archive ?? null, false, existing?.controller?.history); return matchingReceipt(existing, facts, input, assets.bindings, controller) !== null; } catch { return false; } } @@ -190,9 +196,124 @@ export async function assertNativeHarnessFilesystemArchiveSource(db: Db, owner: const facts = await databaseFacts(db, registered.scope, true); if (facts.factsDigest !== registered.factsDigest || facts.factsDigest !== owner.filesystemIndependence.factsDigest || facts.source.id !== owner.runId || facts.anchor.id !== owner.selectedLastRunId) fail(); + const history = await captureAuthorityHistory(db, facts, registered.signal); + if (history?.localDigest !== registered.history?.localDigest || digest(history?.binding ?? null) !== digest(registered.history?.binding ?? null)) fail(); } -async function controllerBinding(db: Db, facts: Awaited>, archive: ControllerBinding["archive"], lock: boolean): Promise { +export function assertNativeHarnessFilesystemArchiveContents(owner: SettledControllerArchiveOwner, archive: { manifestSha256: string }) { + if (evidenceOwners.get(owner)?.archiveManifestSha256 !== archive.manifestSha256) fail(); +} + +async function historyEpochBinding(db: Db, facts: Awaited>, + epoch: Omit & { headerDigest?: string | null }, lock: boolean, contents?: string): Promise { + if (!/^epoch-[a-f0-9]{24}$/.test(epoch.name) || !/^[a-f0-9]{64}$/.test(epoch.identityDigest)) fail(); + const run = facts.runs.find(run => run.id === epoch.runId); + if (!run || run.id === facts.source.id || run.createdAt >= facts.source.createdAt + || basename(scopedRunnerdStateRoot(parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput))) + !== basename(scopedRunnerdStateRoot(facts.execution))) fail(); + const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput), identity = epoch.identity; + if (!identity || identity.runId !== run.id || identity.runnerInstanceId !== run.runnerInstanceId + || identity.normalizedSessionId !== run.nativeSessionId || digest(identity) !== epoch.identityDigest + || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId + || execution.binding.issueId !== run.nativeIssueId || execution.binding.runId !== run.id + || execution.session.normalizedSessionId !== run.nativeSessionId) fail(); + const query = db.select().from(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.companyId, run.companyId), + eq(nativeRunnerCheckpoints.issueId, facts.anchor.nativeIssueId!), eq(nativeRunnerCheckpoints.runId, run.id), + eq(nativeRunnerCheckpoints.part, "header"))); + const [row] = await (lock ? query.for("share") : query); + const header = row && await loadNativeRunnerCheckpointHeader(db, { companyId: run.companyId, issueId: facts.anchor.nativeIssueId!, runId: run.id }); + if (row ? !header || digest(inspectDurablePrpCheckpointHeader(header)) !== epoch.identityDigest + : identity.environmentLeaseId !== execution.binding.executionWorkspaceId) fail(); + const headerDigest = row ? digest(row) : null; + if (epoch.headerDigest !== undefined && epoch.headerDigest !== headerDigest) fail(); + if (contents !== undefined) { + const artifact = header ? JSON.parse(header) : null; + try { + captureLegacyDurablePrpCheckpoint(contents, { runId: run.id, runnerInstanceId: identity.runnerInstanceId, + normalizedSessionId: identity.normalizedSessionId }, artifact ? { version: artifact.runnerVersion, digest: artifact.runnerDigest } : undefined); + } catch (error) { + if (row || !(error instanceof Error) || error.message !== "native_runner_checkpoint_artifact_unproven") throw error; + } + } + return { name: epoch.name, runId: run.id, identity, identityDigest: epoch.identityDigest, headerDigest }; +} + +async function captureAuthorityHistory(db: Db, facts: Awaited>, signal: AbortSignal): Promise { + signal.throwIfAborted(); + const root = scopedRunnerdStateRoot(facts.execution), base = join(root, "authority-epochs"); + const stat = await lstat(base).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (!stat) return undefined; + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() + || await realpath(root) !== root || await realpath(base) !== base) fail(); + const directoryNames = async (path: string) => { + const names: string[] = []; + for await (const entry of await opendir(path)) { + signal.throwIfAborted(); if (names.length >= 128) fail(); names.push(entry.name); + } + return names.sort(); + }; + const names = await directoryNames(base); if (!names.length) return undefined; + const entries: NativeControllerHistoryEntry[] = [], metadata: unknown[] = [], inodes = new Set(); + let bytes = 0; + const visit = async (path: string): Promise => { + signal.throwIfAborted(); + if (entries.length >= 128 || !/^authority-epochs(?:\/epoch-[a-f0-9]{24}(?:\/control-plane(?:\/control-plane-state\.json)?|\/runner-state\.json)?)?$/.test(path)) fail(); + const file = join(root, path), s = await lstat(file); + if (s.isSymbolicLink() || !s.isDirectory() && !s.isFile() || s.dev !== stat.dev || s.uid !== stat.uid || (s.mode & 0o7000) !== 0) fail(); + metadata.push([path, s.dev, s.ino, s.size, s.mode, s.mtimeMs, s.ctimeMs]); + if (s.isDirectory()) { + if (path.endsWith(".json")) fail(); + entries.push({ path, mode: s.mode & 0o777, type: "directory" }); + const children = await directoryNames(file); + for (const name of children) await visit(`${path}/${name}`); + const after = await lstat(file); + if (after.dev !== s.dev || after.ino !== s.ino || after.ctimeMs !== s.ctimeMs + || digest(await directoryNames(file)) !== digest(children)) fail(); + } else { + if (!path.endsWith("/control-plane/control-plane-state.json") && !path.endsWith("/runner-state.json")) fail(); + const key = `${s.dev}:${s.ino}`; + if (s.nlink !== 1 || inodes.has(key) || s.size > 16 * 1024 * 1024 || (bytes += s.size) > 64 * 1024 * 1024) fail(); + inodes.add(key); + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const opened = await handle.stat(); + if (!opened.isFile() || opened.dev !== s.dev || opened.ino !== s.ino || opened.size !== s.size) fail(); + const buffer = Buffer.alloc(s.size + 1); let length = 0; + while (length < buffer.length) { + signal.throwIfAborted(); + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length); + if (!bytesRead) break; length += bytesRead; + } + if (length !== s.size) fail(); + const contents = buffer.subarray(0, length), after = await handle.stat(), named = await lstat(file); + if (after.size !== s.size || after.ctimeMs !== s.ctimeMs || after.mtimeMs !== s.mtimeMs + || named.dev !== s.dev || named.ino !== s.ino || !Buffer.from(contents.toString("utf8")).equals(contents)) fail(); + entries.push({ path, mode: s.mode & 0o777, type: "file", content: contents.toString("utf8") }); + } finally { await handle.close(); } + } + }; + await visit("authority-epochs"); + const epochs: HistoryEpoch[] = []; + for (const name of names) { + const file = entries.find(entry => entry.path === `authority-epochs/${name}/control-plane/control-plane-state.json`); + if (!file || file.type !== "file") fail(); + const value = JSON.parse(file.content), identity = value.identity; + if (!identity || name !== `epoch-${sha(JSON.stringify(identity)).slice(0, 24)}`) fail(); + const epoch = await historyEpochBinding(db, facts, { name, runId: identity.runId, identity, identityDigest: digest(identity) }, false, file.content); + const runner = entries.find(entry => entry.path === `authority-epochs/${name}/runner-state.json`); + if (runner) { + if (runner.type !== "file") fail(); + const state = JSON.parse(runner.content); + if (state.schema !== "paperclip.runner.durable.state.v1" || state.lifecycle !== "suspended" + || Object.entries(identity).some(([key, value]) => state[key] !== value)) fail(); + } + epochs.push(epoch); + } + return { entries, binding: { entries: nativeControllerHistoryManifest(entries), epochs }, localDigest: digest(metadata) }; +} + +async function controllerBinding(db: Db, facts: Awaited>, archive: ControllerBinding["archive"], lock: boolean, + history?: HistoryBinding): Promise { const partsQuery = db.select().from(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.companyId, facts.anchor.companyId), eq(nativeRunnerCheckpoints.issueId, facts.anchor.nativeIssueId!), eq(nativeRunnerCheckpoints.runId, facts.source.id))) .orderBy(asc(nativeRunnerCheckpoints.part)); @@ -204,17 +325,35 @@ async function controllerBinding(db: Db, facts: Awaited 32 + || new Set(history.epochs.map(epoch => epoch.name)).size !== history.epochs.length) fail(); + const paths = new Set(["authority-epochs"]); + for (const epoch of history.epochs) { + if (digest(await historyEpochBinding(db, facts, epoch, lock)) !== digest(epoch)) fail(); + for (const suffix of ["", "/control-plane", "/control-plane/control-plane-state.json"]) paths.add(`authority-epochs/${epoch.name}${suffix}`); + if (history.entries.some(entry => entry.path === `authority-epochs/${epoch.name}/runner-state.json`)) paths.add(`authority-epochs/${epoch.name}/runner-state.json`); + } + if (paths.size !== history.entries.length || history.entries.some(entry => !paths.has(entry.path) + || entry.type !== (entry.path.endsWith(".json") ? "file" : "directory"))) fail(); + } + const descriptor = nativeControllerEvidenceDescriptor({ companyId: facts.anchor.companyId, issueId: facts.anchor.nativeIssueId!, + runId: facts.source.id, controllerDigest: archive.controllerDigest, supervisorDigest: archive.supervisorDigest, + purpose: "filesystem_independence", history: history?.entries, historyBindingDigest: history ? digest(history) : undefined }); + if (row.scopeDigest !== descriptor.scopeDigest || row.manifestSha256 !== descriptor.manifestSha256) fail(); + } else if (history) { + fail(); } - return { checkpointPartsDigest: digest(parts), archive }; + return { checkpointPartsDigest: digest(parts), archive, ...(history ? { history } : {}) }; } async function preserveControllerEvidence(db: Db, facts: Awaited>, input: Scope & { source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; -}): Promise { +}, history?: CapturedHistory): Promise { const material = await readLegacyNativeControllerArchiveSource(facts.source); if (material.paths.root !== scopedRunnerdStateRoot(facts.execution)) fail(); if (material.controller === undefined) { - if (material.supervisor !== undefined) fail(); + if (material.supervisor !== undefined || history) fail(); return controllerBinding(db, facts, null, false); } const identity = JSON.parse(material.controller).identity as Record | undefined; @@ -238,11 +377,13 @@ async function preserveControllerEvidence(db: Db, facts: Awaited>, assets: Awaited>, storage: StorageService, db: Db, verifyStorage: boolean, signal: AbortSignal, controller: ControllerBinding, plans = new Map()) { const root = scopedRunnerdStateRoot(facts.execution), metadata: unknown[] = [], store = skillPackageStore(db, storage); + const history = await captureAuthorityHistory(db, facts, signal); + if (digest(history?.binding ?? null) !== digest(controller.history ?? null)) fail(); + if (history) metadata.push({ history: history.localDigest }); const directories = resolveNativeHarnessPersistenceProfile(facts.execution).directories; const stat = async (path: string) => lstat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); const rootStat = await stat(root); @@ -288,7 +432,7 @@ async function localFacts(facts: Awaited>, asse const control = await stat(join(root, "control-plane")); if (control && (!control.isDirectory() || control.isSymbolicLink() || (await readdir(join(root, "control-plane"))).some(name => name !== "control-plane-state.json"))) fail(); - for (const name of ["continuity-breaks", "authority-epochs", "provider-artifacts", "failover-backups"]) { + for (const name of ["continuity-breaks", "provider-artifacts", "failover-backups"]) { const entry = await stat(join(root, name)); if (entry && (!entry.isDirectory() || entry.isSymbolicLink() || (await readdir(join(root, name))).length)) fail(); } @@ -378,7 +522,8 @@ export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); if (input.storage.provider !== "s3") fail(); const facts = await databaseFacts(db, input, false), assets = await assetBindings(db, facts.anchor, facts.source.id); - const controller = await preserveControllerEvidence(db, facts, input); + const history = await captureAuthorityHistory(db, facts, input.signal); + const controller = await preserveControllerEvidence(db, facts, input, history); const local = await localFacts(facts, assets, input.storage, db, true, input.signal, controller), localFactsDigest = local.digest; const previous = facts.anchor.runnerProfileJson?.[KEY]; if (previous != null) { @@ -403,7 +548,7 @@ export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: .where(eq(companySkillRepositorySnapshots.id, row.id)).for("update"); } const currentAssets = await assetBindings(tx, current.anchor, current.source.id); - const currentController = await controllerBinding(tx, current, controller.archive, true); + const currentController = await controllerBinding(tx, current, controller.archive, true, controller.history); if (digest(currentAssets.bindings) !== receipt.assetsDigest || digest(currentController) !== digest(controller) || (await localFacts(current, currentAssets, input.storage, tx, false, input.signal, controller, local.plans)).digest !== localFactsDigest) fail();