diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 9b491cdf608..af376af072d 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -122,8 +122,8 @@ original-volume anchor or filesystem read. These validations add bounded per-retained-scope database reads during startup. They do not add a worker or a poll. Missing or invalid byte evidence remains an enrollment blocker. -The original-volume owner can refresh a WEB receipt whose database facts digest -has become stale only when fresh source inspection, current-key checkpoint +The original-volume owner can refresh a WEB or full native receipt whose database +facts digest has become stale only when fresh source inspection, current-key checkpoint validation and ciphertext hash/size readback prove the preserved bindings unchanged. It reuses the same archive and takes the existing finalizer, task and run fences before comparing the prior receipt digest and replacing it. The company activity log @@ -133,6 +133,10 @@ the exact prior proof from the unchanged current bindings before publication. No enter the audit delta; normal activity redaction remains enabled. Changed evidence, concurrent invalid receipts and active sources remain blocked; remote recovery obligations are unchanged. +Full native reproof still requires canonical destroyed-provider receipts. A +database-only legacy workspace migration marker can stale the facts digest while +the preserved controller, skill assets and local byte proof remain unchanged; +refresh authenticates that same object and does not upload a replacement archive. Local inode and timestamp observations may change between preservation attempts without changing byte coverage. Valid receipts remain idempotent after those diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts index 0c6a62c4d30..3d25873d71c 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts @@ -34,6 +34,7 @@ import { nativeCheckpointRetirement } from "./native-runner-checkpoint-retiremen import { nativeSha256 } from "./canonical.js"; import { nativeControllerEvidenceDescriptor, NATIVE_CONTROLLER_WEB_LIMITS, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; import { logActivity } from "../activity-log.js"; +import { LEGACY_WORKSPACE_RECOVERY_MIGRATION } from "../legacy-workspace-recovery-migration.js"; vi.mock("../../build-commit.js", () => ({ readBuildCommit: () => "a".repeat(40) })); @@ -389,6 +390,130 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); }); + async function staleNativeFixture(checkpointParts?: number) { + const f = await controllerFixture(true, "unversioned", checkpointParts); + const first = await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const previous = saved!.runnerProfileJson!.nativeHarnessFilesystemIndependence as Record; + const marker = { schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: randomUUID(), + workspaceId: f.execution.binding.executionWorkspaceId }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...saved!.runnerProfileJson, + legacyWorkspaceRecoveryMigration: marker }, updatedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + return { ...f, first, previous }; + } + + it("reproves stale native facts after a legacy migration marker without replacing preserved evidence", async () => { + const f = await staleNativeFixture(453), { first, previous } = f; + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId)); + const leases = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId)); + const finalizers = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.input.companyId)); + const results = await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.input.companyId)); + const parts = await db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)); + const backups = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + const packages = await db.select().from(companySkillRepositorySnapshots).where(eq(companySkillRepositorySnapshots.companyId, f.input.companyId)); + const versions = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, f.input.companyId)); + const put = vi.spyOn(f.storage, "putFile"), get = vi.spyOn(f.storage, "getObject"); + const result = await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = current!.runnerProfileJson!.nativeHarnessFilesystemIndependence as Record; + expect(receipt).toEqual({ ...previous, factsDigest: expect.any(String), observedAt: expect.any(String) }); + expect(receipt.factsDigest).not.toBe(previous.factsDigest); + expect(result.receiptDigest).not.toBe(first.receiptDigest); + expect(put).not.toHaveBeenCalled(); expect(get).toHaveBeenCalled(); + expect(await readFile(f.controllerPath, "utf8")).toBe(f.raw); + expect(await readFile(f.supervisorPath, "utf8")).toBe(f.supervisor); + expect(parts).toHaveLength(453); + expect(await db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual(parts); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId))).toEqual(backups); + expect(await db.select().from(companySkillRepositorySnapshots).where(eq(companySkillRepositorySnapshots.companyId, f.input.companyId))).toEqual(packages); + expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, f.input.companyId))).toEqual(versions); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId))).toEqual(leases); + expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.input.companyId))).toEqual(finalizers); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.input.companyId))).toEqual(results); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId))) + .toEqual(runs.map(run => ({ ...run, runnerProfileJson: { ...run.runnerProfileJson, nativeHarnessFilesystemIndependence: receipt } }))); + const audits = await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified"))); + expect(audits).toHaveLength(2); + const audit = audits.find(row => row.details?.previousReceiptDigest === first.receiptDigest)!.details!; + expect(nativeSha256({ ...receipt, ...audit.previousReceiptState as Record })).toBe(first.receiptDigest); + expect((await preserveNativeHarnessFilesystemIndependence(db, f.input)).receiptDigest).toBe(result.receiptDigest); + expect(put).not.toHaveBeenCalled(); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-native-reproof")); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-native-reproof/state")); vi.stubEnv("RAILWAY_VOLUME_ID", ""); + get.mockRejectedValue(new Error("DB-only validation must not read storage")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + }, 30_000); + + it.each(["assets-binding", "raw-controller", "termination", "unknown-field"])( + "retains stale native proof when reproof rejects %s", async control => { + const f = await staleNativeFixture(); + if (control === "raw-controller") { + const changed = f.raw.replace("Raw stream bytes absent from control-v1 records", "Changed raw streaming evidence"); + expect(changed).not.toBe(f.raw); await writeFile(f.controllerPath, changed); + } else if (control === "termination") { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const metadata = { ...lease!.metadata }; delete metadata.remoteExecutionTermination; + await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.leaseId)); + } else { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = { ...f.previous, ...(control === "assets-binding" ? { assetsDigest: "0".repeat(64) } : { unknownEvidence: "retain" }) }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run!.runnerProfileJson, + nativeHarnessFilesystemIndependence: receipt } }).where(eq(heartbeatRuns.id, f.runId)); + } + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId)); + const leases = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId)); + const backups = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + const put = vi.spyOn(f.storage, "putFile"); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("unverified"); + expect(put).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId))).toEqual(runs); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId))).toEqual(leases); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId))).toEqual(backups); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + }); + + it.each(["matching-winner", "changed-receipt", "active-owner"])( + "guards stale native reproof publication after %s", async control => { + const f = await staleNativeFixture(); + let release!: () => void, entered!: () => void, delay = true; + const blocked = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { entered = resolve; }); + const get = f.storage.getObject.bind(f.storage), put = vi.spyOn(f.storage, "putFile"); + vi.spyOn(f.storage, "getObject").mockImplementation(async (...args) => { + if (delay) { delay = false; entered(); await blocked; } + return get(...args); + }); + const late = preserveNativeHarnessFilesystemIndependence(db, f.input); + try { + await ready; + const winner = control === "matching-winner" ? await preserveNativeHarnessFilesystemIndependence(db, f.input) : null; + if (control === "changed-receipt") { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run!.runnerProfileJson, + nativeHarnessFilesystemIndependence: { ...f.previous, observedAt: new Date(0).toISOString() } } }) + .where(eq(heartbeatRuns.id, f.runId)); + } + if (control === "active-owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "new-owner", + leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId)); + const finalizers = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.input.companyId)); + const backups = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + release(); + if (winner) expect((await late).receiptDigest).toBe(winner.receiptDigest); + else await expect(late).rejects.toThrow("unverified"); + expect(put).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId))).toEqual(runs); + expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.input.companyId))).toEqual(finalizers); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId))).toEqual(backups); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(winner ? 2 : 1); + } finally { release(); await late.catch(() => undefined); } + }); + async function staleWebFixture() { const f = await webFixture(), first = await preserveNativeHarnessFilesystemIndependence(db, f.input); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.ts index b818e32d760..706070a1531 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.ts @@ -781,17 +781,27 @@ export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: if (input.storage.provider !== "s3") fail(); if (input.mode === "web_evidence") return preserveWebEvidence(db, input); const facts = await databaseFacts(db, input, false), assets = await assetBindings(db, facts.anchor, facts.source.id); + const previous = facts.anchor.runnerProfileJson?.[KEY] as Receipt | undefined; + const previousDigest = previous == null ? null : digest(previous); + if (previous != null && (previous.schema !== SCHEMA || !previous.controller)) fail(); const history = await captureAuthorityHistory(db, facts, input.signal); - const controller = await preserveControllerEvidence(db, facts, input, history); + const controller = await preserveControllerEvidence(db, facts, input, history, undefined, previous?.controller); const local = await localFacts(facts, assets, input.storage, db, true, input.signal, controller), localFactsDigest = local.digest; - const previous = facts.anchor.runnerProfileJson?.[KEY]; if (previous != null) { - if (!await hasValidNativeHarnessFilesystemIndependence(db, input)) fail(); - return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(previous) }; + if (!/^[a-f0-9]{64}$/.test(previous.factsDigest) + || !matchingReceipt(previous, facts, input, assets.bindings, controller, undefined, previous.factsDigest)) fail(); + if (matchingReceipt(previous, facts, input, assets.bindings, controller)) { + if (!await hasValidNativeHarnessFilesystemIndependence(db, input)) fail(); + await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); + return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: previousDigest! }; + } } const receipt: Receipt = { schema: SCHEMA, selectedSessionId: input.selectedSessionId, scopeDigest: input.scopeDigest, lastRunId: facts.anchor.id, sourceRunId: facts.source.id, factsDigest: facts.factsDigest, sourceCapabilityFingerprint: input.source.fingerprint, localFactsDigest, assetsDigest: digest(assets.bindings), ...assets.bindings, controller, observedAt: new Date().toISOString() }; + const previousReceiptState = previous ? { factsDigest: previous.factsDigest, localFactsDigest: previous.localFactsDigest, + sourceCapabilityFingerprint: previous.sourceCapabilityFingerprint, observedAt: previous.observedAt } : null; + if (previousReceiptState && digest({ ...receipt, ...previousReceiptState }) !== previousDigest) fail(); let publishedReceipt = receipt; const publications: ActivityPublication[] = []; await db.transaction(async transaction => { @@ -815,16 +825,16 @@ export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: const existing = current.anchor.runnerProfileJson?.[KEY]; if (existing != null) { const winner = matchingReceipt(existing, current, input, currentAssets.bindings, currentController); - if (!winner) fail(); - publishedReceipt = winner; - return; + if (winner) { publishedReceipt = winner; return; } } + if ((existing == null ? null : digest(existing)) !== previousDigest) fail(); await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...current.anchor.runnerProfileJson, [KEY]: receipt } }) .where(and(eq(heartbeatRuns.id, current.anchor.id), eq(heartbeatRuns.companyId, input.companyId))); await logActivity(tx, { companyId: input.companyId, actorType: "system", actorId: "native-harness-preservation", action: "native_harness_filesystem_independence_verified", entityType: "heartbeat_run", entityId: current.anchor.id, agentId: current.anchor.agentId, runId: current.anchor.id, details: { scopeDigest: input.scopeDigest, - sourceRunId: current.source.id, sourceCapabilityFingerprint: receipt.sourceCapabilityFingerprint, factsDigest: receipt.factsDigest } }, publications); + sourceRunId: current.source.id, sourceCapabilityFingerprint: receipt.sourceCapabilityFingerprint, factsDigest: receipt.factsDigest, + ...(previousReceiptState ? { previousReceiptDigest: previousDigest, previousReceiptState } : {}) } }, publications); }); for (const publication of publications) publishActivity(publication); return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(publishedReceipt) };