diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 69b3adba55a..bb1594be275 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -91,8 +91,34 @@ validate the published receipt using only database bindings, without the old volume, a fresh source witness, or storage reads. Changed or missing bindings invalidate it. Existing resume and replacement policies remain authoritative, including governed-wait failures and warm-session refusals. No provider history -is reconstructed or declared recovered, and uncertain provider sources remain -enrollment blockers. +is reconstructed or declared recovered. + +The same original-volume maintenance owner has a separate `web_evidence` mode. +It stores a `paperclip.native-harness-web-evidence.v1` receipt. This mode proves +that the web server's retained files have durable byte coverage. It does not +require an external VM's termination receipt and does not settle that VM's +recovery obligation. It requires the same settled web controller, finalizer, +task and physical-writer fences. The encrypted `original_volume_source_evidence` +archive includes a closed inventory of all nonexcluded files under the canonical +session root, the separate warm supervisor, and each admitted skill manifest. +Files retain their bytes and modes, including local edits and a skill whose +version row was deleted. Such a skill remains forensic evidence: the owner does +not recreate its version or make it executable. Canonical credential exclusions +stay excluded. Unknown directories, symlinks, incomplete manifests, active web +writers and changed source facts fail closed. The same 128-entry, 16-MiB-file and +64-MiB-total limits apply independently to this inventory. + +A genuinely absent raw controller stays absent. A supervisor-only archive still +requires the exact durable controller checkpoint and source identity. Old +terminated-provider receipts and history-free manifests keep their existing +shape. External lease status and cleanup observations can change without +creating a new web filesystem dependency. Allocation, stamp, selected source, +checkpoint and admitted-input changes invalidate coverage. VM references, +recovery actions, warnings and continuation policies stay unchanged. Replacement +validation uses only the published archive and database bindings; it needs no +original-volume anchor or filesystem read. These validations add bounded +per-retained-scope database reads during startup. They do not add a worker or a +poll. Missing or invalid byte evidence remains an enrollment blocker. Receipt validation adds bounded per-scope database reads to startup eligibility. It does not add a recurring scan or network work. Qualification must measure the diff --git a/server/src/services/native-runtime/native-controller-archive-source.ts b/server/src/services/native-runtime/native-controller-archive-source.ts index 7f55ed1acf1..43f3b604002 100644 --- a/server/src/services/native-runtime/native-controller-archive-source.ts +++ b/server/src/services/native-runtime/native-controller-archive-source.ts @@ -13,7 +13,7 @@ export type SettledControllerArchiveOwner = { kind: "settled_controller_archive"; companyId: string; issueId: string; agentId: string; runId: string; sessionId: string; selectedSessionId: string; selectedLastRunId: string; - profileDigest: string; coordinatorDigest: string; controllerDigest: string; supervisorDigest: string | null; + profileDigest: string; coordinatorDigest: string; controllerDigest: string | null; supervisorDigest: string | null; filesystemIndependence?: { source: OriginalHarnessVolumeCapability; scopeDigest: string; factsDigest: string }; }; const fileDigest = (value: string) => createHash("sha256").update(value).digest("hex"); @@ -33,6 +33,7 @@ export function nativeControllerArchiveCoordinatorDigest(value: { export async function assertSettledControllerArchiveSource(db: Db, owner: SettledControllerArchiveOwner) { function fail(): never { throw new Error("native_controller_archive_source_unsettled"); } + if (owner.controllerDigest === null && !owner.filesystemIndependence) fail(); if (owner.filesystemIndependence) { await assertOriginalHarnessVolumeSource(owner.filesystemIndependence.source); await (await import("./native-harness-filesystem-independence.js")).assertNativeHarnessFilesystemArchiveSource(db, owner); @@ -68,7 +69,7 @@ export async function assertSettledControllerArchiveSource(db: Db, owner: Settle normalizedSessionId: owner.sessionId, beforeCreatedAt: selectedRun.createdAt, includeProgressBarrier: true }); if (!selected || selected.id !== owner.runId || nativeControllerArchiveProfileDigest(selected.runnerProfileJson) !== owner.profileDigest) fail(); const material = await readLegacyNativeControllerArchiveDigests(selected); - if (material.controllerDigest !== owner.controllerDigest + if ((material.controllerDigest ?? null) !== owner.controllerDigest || (material.supervisorDigest ?? null) !== owner.supervisorDigest) fail(); } diff --git a/server/src/services/native-runtime/native-controller-archive.ts b/server/src/services/native-runtime/native-controller-archive.ts index b92cad0b6e2..b698c49ba6e 100644 --- a/server/src/services/native-runtime/native-controller-archive.ts +++ b/server/src/services/native-runtime/native-controller-archive.ts @@ -7,25 +7,36 @@ import type { StorageService } from "../../storage/types.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import type { SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; +import { assertSkillSnapshotPath } from "../skill-snapshot.js"; export type NativeControllerHistoryEntry = { path: string; mode: number; type: "directory" } | { path: string; mode: number; type: "file"; content: string }; export type NativeControllerHistoryManifestEntry = { path: string; mode: number; type: "directory" } | { path: string; mode: number; type: "file"; size: number; sha256: string }; -type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string; +export type NativeControllerWebEntry = { path: string; mode: number; type: "directory" } + | { path: string; mode: number; type: "file"; content: Buffer }; +type WebEvidence = { entries: readonly T[]; bindingDigest: string }; +type Source = { companyId: string; issueId: string; runId: string; controller?: string; supervisor?: string; history?: readonly NativeControllerHistoryEntry[]; historyBindingDigest?: string; + web?: WebEvidence; purpose?: "filesystem_independence" }; -const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); +const sha256 = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); export function nativeControllerHistoryManifest(history: readonly NativeControllerHistoryEntry[]): NativeControllerHistoryManifestEntry[] { return history.map(entry => entry.type === "directory" ? { ...entry } : { path: entry.path, mode: entry.mode, type: entry.type, size: Buffer.byteLength(entry.content), sha256: sha256(entry.content) }); } +export function nativeControllerWebManifest(entries: readonly NativeControllerWebEntry[]): NativeControllerHistoryManifestEntry[] { + return entries.map(entry => entry.type === "directory" ? { ...entry } + : { path: entry.path, mode: entry.mode, type: entry.type, size: entry.content.length, sha256: sha256(entry.content) }); +} + export function nativeControllerEvidenceDescriptor(source: { companyId: string; issueId: string; runId: string; - controllerDigest: string; supervisorDigest: string | null; history?: readonly NativeControllerHistoryManifestEntry[]; + controllerDigest: string | null; supervisorDigest: string | null; history?: readonly NativeControllerHistoryManifestEntry[]; historyBindingDigest?: string; + web?: WebEvidence; purpose?: "filesystem_independence" }) { const history = source.history?.length ? source.history.map(entry => entry.type === "directory" ? { path: entry.path, mode: entry.mode, type: entry.type } @@ -49,20 +60,45 @@ export function nativeControllerEvidenceDescriptor(source: { companyId: string; } } } + const web = source.web ? { bindingDigest: source.web.bindingDigest, + entries: source.web.entries.map(entry => entry.type === "directory" ? { path: entry.path, mode: entry.mode, type: entry.type } + : { path: entry.path, mode: entry.mode, type: entry.type, size: entry.size, sha256: entry.sha256 }) } : undefined; + if (source.controllerDigest === null && !web) throw new Error("native_controller_archive_authority_invalid"); + if (web) { + const paths = new Set(); let bytes = 0; + if (source.purpose !== "filesystem_independence" || web.entries.length > 128 || !/^[a-f0-9]{64}$/.test(web.bindingDigest)) { + throw new Error("native_controller_archive_authority_invalid"); + } + for (const entry of web.entries) { + assertSkillSnapshotPath(entry.path); + if (!/^(?:(web-home|web-state|web-manifests)(\/|$)|web-supervisor\.json$)/.test(entry.path) || paths.has(entry.path) + || !["directory", "file"].includes(entry.type) + || !Number.isInteger(entry.mode) || entry.mode < 0 || (entry.mode & ~0o777) !== 0 + || entry.path.includes("/") && !paths.has(entry.path.slice(0, entry.path.lastIndexOf("/"))) + || entry.type === "file" && (!Number.isSafeInteger(entry.size) || entry.size < 0 || entry.size > 16 * 1024 * 1024 + || !/^[a-f0-9]{64}$/.test(entry.sha256) || (bytes += entry.size) > 64 * 1024 * 1024)) { + throw new Error("native_controller_archive_authority_invalid"); + } + paths.add(entry.path); + } + } const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId, issueId: source.issueId, runId: source.runId, reason: source.purpose === "filesystem_independence" ? "original_volume_source_evidence" : "original_runner_artifact_unproven", - files: { "control-plane-state.json": source.controllerDigest, + files: { ...(source.controllerDigest === null ? {} : { "control-plane-state.json": source.controllerDigest }), ...(source.supervisorDigest === null ? {} : { "supervisor.json": source.supervisorDigest }), - ...(history ? Object.fromEntries(history.filter(entry => entry.type === "file").map(entry => [entry.path, entry.sha256])) : {}) }, - ...(history ? { history, historyBindingDigest: source.historyBindingDigest } : {}) }); + ...(history ? Object.fromEntries(history.filter(entry => entry.type === "file").map(entry => [entry.path, entry.sha256])) : {}), + ...(web ? Object.fromEntries(web.entries.filter(entry => entry.type === "file").map(entry => [entry.path, entry.sha256])) : {}) }, + ...(history ? { history, historyBindingDigest: source.historyBindingDigest } : {}), + ...(web ? { web: { entries: web.entries, bindingDigest: web.bindingDigest, controllerAbsent: source.controllerDigest === null } } : {}) }); return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` }; } export function legacyNativeControllerArchive(source: Source) { - return nativeControllerEvidenceDescriptor({ ...source, controllerDigest: sha256(source.controller), + return nativeControllerEvidenceDescriptor({ ...source, controllerDigest: source.controller === undefined ? null : sha256(source.controller), supervisorDigest: source.supervisor === undefined ? null : sha256(source.supervisor), - history: source.history ? nativeControllerHistoryManifest(source.history) : undefined }); + history: source.history ? nativeControllerHistoryManifest(source.history) : undefined, + web: source.web ? { entries: nativeControllerWebManifest(source.web.entries), bindingDigest: source.web.bindingDigest } : undefined }); } export async function hasArchivedLegacyNativeController(db: Db, input: Source & { storage: StorageService; signal?: AbortSignal }) { @@ -73,7 +109,7 @@ export async function hasArchivedLegacyNativeController(db: Db, input: Source & export async function archiveFailedLegacyNativeController(db: Db, input: Source & { storage: StorageService; writersStopped?: true; onlineOwner?: SettledControllerArchiveOwner; signal: AbortSignal; }) { - if (input.purpose || input.history?.length || input.onlineOwner?.filesystemIndependence) throw new Error("native_controller_archive_authority_invalid"); + if (input.controller === undefined || input.web || input.purpose || input.history?.length || input.onlineOwner?.filesystemIndependence) throw new Error("native_controller_archive_authority_invalid"); return persistLegacyControllerEvidence(db, input); } @@ -83,7 +119,8 @@ export async function archiveNativeControllerSourceEvidence(db: Db, input: Sourc if (input.purpose !== "filesystem_independence" || !input.onlineOwner.filesystemIndependence) { throw new Error("native_controller_archive_authority_invalid"); } - input = { ...input, history: input.history?.map(entry => ({ ...entry })) }; + input = { ...input, history: input.history?.map(entry => ({ ...entry })), web: input.web ? { ...input.web, + entries: input.web.entries.map(entry => entry.type === "file" ? { ...entry, content: Buffer.from(entry.content) } : { ...entry }) } : undefined }; (await import("./native-harness-filesystem-independence.js")).assertNativeHarnessFilesystemArchiveContents( input.onlineOwner, legacyNativeControllerArchive(input)); await db.transaction(async tx => (await import("./native-controller-archive-source.js")) @@ -96,7 +133,7 @@ async function persistLegacyControllerEvidence(db: Db, input: Source & { }) { if (input.onlineOwner ? input.writersStopped || input.onlineOwner.kind !== "settled_controller_archive" || input.onlineOwner.companyId !== input.companyId || input.onlineOwner.issueId !== input.issueId - || input.onlineOwner.runId !== input.runId || input.onlineOwner.controllerDigest !== sha256(input.controller) + || input.onlineOwner.runId !== input.runId || input.onlineOwner.controllerDigest !== (input.controller === undefined ? null : sha256(input.controller)) || input.onlineOwner.supervisorDigest !== (input.supervisor === undefined ? null : sha256(input.supervisor)) : !input.writersStopped) { throw new Error("native_controller_archive_authority_invalid"); } @@ -105,13 +142,18 @@ async function persistLegacyControllerEvidence(db: Db, input: Source & { const root = await mkdtemp(join(tmpdir(), "pc-legacy-controller-archive-")); try { await writeFile(join(root, "manifest.json"), descriptor.manifest, { mode: 0o600 }); - await writeFile(join(root, "control-plane-state.json"), input.controller, { mode: 0o600 }); + if (input.controller !== undefined) await writeFile(join(root, "control-plane-state.json"), input.controller, { mode: 0o600 }); if (input.supervisor !== undefined) await writeFile(join(root, "supervisor.json"), input.supervisor, { mode: 0o600 }); for (const entry of input.history ?? []) { if (entry.type === "directory") await mkdir(join(root, entry.path), { mode: 0o700 }); else await writeFile(join(root, entry.path), entry.content, { mode: entry.mode }); } for (const entry of [...(input.history ?? [])].reverse()) await chmod(join(root, entry.path), entry.mode); + for (const entry of input.web?.entries ?? []) { + if (entry.type === "directory") await mkdir(join(root, entry.path), { mode: 0o700 }); + else await writeFile(join(root, entry.path), entry.content, { mode: entry.mode }); + } + for (const entry of [...(input.web?.entries ?? [])].reverse()) await chmod(join(root, entry.path), entry.mode); await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest: descriptor.scopeDigest, signal: input.signal, owner: input.onlineOwner ?? { kind: "offline_migration", writersStopped: true, terminalFailureOnly: true, companyId: input.companyId, issueId: input.issueId, runId: input.runId } }); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts index 38a99f5c080..d1640ebb7e9 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, truncate, writeFile } from "node:fs/promises"; +import { chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, symlink, truncate, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { and, eq } from "drizzle-orm"; @@ -43,7 +43,7 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await db.delete(agentTaskSessions); for (const root of roots.splice(0)) await removeRuntimeAssetTree(root); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); - async function fixture(kind: "unversioned" | "repository" | "package" | "none" = "unversioned", managedWorkspace = false) { + async function fixture(kind: "unversioned" | "repository" | "package" | "deleted" | "none" = "unversioned", managedWorkspace = false) { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); const mount = await realpath(await mkdtemp(join(tmpdir(), "native-harness-original-"))); roots.push(mount); const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); @@ -60,9 +60,10 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" mountIdentity: { dev: m.dev, ino: m.ino }, instanceIdentity: { dev: instance.dev, ino: instance.ino }, createdFrom: { deploymentId: randomUUID(), commit: "c".repeat(40), providerWitnessSha256: "d".repeat(64) }, createdAt: new Date().toISOString() }; const bytes = Buffer.from(JSON.stringify(anchor)); await writeFile(originalHarnessVolumeAnchorPath(), bytes, { mode: 0o600 }); + const observed = Date.now(); const witness: ReviewedOriginalHarnessVolumeWitness = { projectId, serviceId, environmentId, volumeId, volumeInstanceId: anchor.volumeInstanceId, mountPath: mount, currentDeploymentId: deploymentId, currentCommit: "a".repeat(40), currentMountIdentity: anchor.mountIdentity, - currentInstanceIdentity: anchor.instanceIdentity, observedAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 120_000).toISOString(), + currentInstanceIdentity: anchor.instanceIdentity, observedAt: new Date(observed).toISOString(), expiresAt: new Date(observed + 120_000).toISOString(), anchorSha256: createHash("sha256").update(bytes).digest("hex"), providerWitnessSha256: "e".repeat(64) }; const source = await verifyOriginalHarnessVolumeSource(witness); const storage = createStorageService({ ...createLocalDiskStorageProvider(join(mount, "objects")), id: "s3" }); @@ -72,14 +73,16 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" const runtimeContext = nativeRuntimeContextFixture(); if (kind !== "none") { let versionId: string | null = null; - const files: CompanySkillVersionFileInventoryEntry[] = [{ path: "SKILL.md", kind: "skill", content: "# original immutable skill\n" }, - { path: "tool.sh", kind: "other" as const, content: "#!/bin/sh\nexit 0\n", executable: true }]; + const files: CompanySkillVersionFileInventoryEntry[] = kind === "deleted" + ? [{ path: "SKILL.md", kind: "skill", content: "# Historical acceptance skill\n\n".padEnd(281, "x") }] + : [{ path: "SKILL.md", kind: "skill", content: "# original immutable skill\n" }, + { path: "tool.sh", kind: "other" as const, content: "#!/bin/sh\nexit 0\n", executable: true }]; let projected = files; if (kind === "repository") { files[0]!.content = "---\nname: retained\ndescription: retained\n---\n# original immutable skill\n"; projected = repositorySkillProjection(files, "SKILL.md"); } - if (kind === "repository" || kind === "package") { + if (kind === "repository" || kind === "package" || kind === "deleted") { const packageId = await skillPackageStore(db, storage).prepare(companyId, files, true); const [skill] = await db.insert(companySkills).values({ companyId, key: "retained", slug: "retained", name: "Retained", markdown: projected[0]!.content }).returning(); const [version] = await db.insert(companySkillVersions).values({ companyId, companySkillId: skill!.id, revisionNumber: 1, @@ -125,8 +128,8 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" return { input, runId, issueId, agentId, leaseId, execution, witness, storage, mount, sourceRoot: scopedRunnerdStateRoot(execution) }; } - async function controllerFixture(managedWorkspace = false) { - const f = await fixture("unversioned", managedWorkspace); + async function controllerFixture(managedWorkspace = false, kind: Parameters[0] = "unversioned", checkpointParts?: number) { + const f = await fixture(kind, managedWorkspace); const completion = await ensureNativeCompletionContract({ db, companyId: f.input.companyId, issue: { id: f.issueId, title: "Retain controller evidence", description: null }, actorId: f.agentId }); const execution = parseNativeExecutionInput({ ...f.execution, session: { ...f.execution.session, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } }, @@ -152,6 +155,13 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" { sourceSeq: 2, sourceEventId: "raw-terminal", eventType: "run.terminal", priority: 0, envelope: { ...identity }, deliveryCount: 1, logicalEffectCount: 1 }); control.store.state.ackedSourceSeq = 2; + if (checkpointParts) { + const artifact = { version: "raw-evidence-fixture", digest: `sha256:${"a".repeat(64)}` }; + for (let count = 0; count < 512 && captureLegacyDurablePrpCheckpoint(JSON.stringify(control.store.state), identity, artifact).records.size < checkpointParts; count++) { + Object.assign(control.queueCommand("runner.suspend"), { status: "completed", result: { status: "completed" } }); + } + expect(captureLegacyDurablePrpCheckpoint(JSON.stringify(control.store.state), identity, artifact).records.size).toBe(checkpointParts); + } const raw = JSON.stringify(control.store.state); control.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, { companyId: f.input.companyId, issueId: f.issueId, runId: f.runId, authorityRunId: f.runId, bootId, generation: 1, leaseOwner })); @@ -174,6 +184,187 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" return { ...f, execution, controllerPath, raw, supervisor, supervisorPath: descriptor.supervisorPath, identity }; } + it("preserves WEB-only evidence with a succeeded source and unresolved external lease", async () => { + const f = await controllerFixture(true, "deleted"); + if (!("runtimeContext" in f.execution)) throw new Error("fixture runtime context missing"); + const versionId = f.execution.runtimeContext.skills[0]!.versionId!; + await db.delete(companySkillVersions).where(eq(companySkillVersions.id, versionId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const metadata = { ...lease!.metadata }; delete metadata.remoteExecutionTermination; + await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.leaseId)); + const beforeLeases = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId)); + const beforeRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId)); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("unverified"); + const result = await preserveNativeHarnessFilesystemIndependence(db, { ...f.input, mode: "web_evidence" }); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(await hasLocalNativeHarnessObligations(db)).toBe(false); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = run!.runnerProfileJson!.nativeHarnessFilesystemIndependence as { schema: string; controller: { archive: { id: string } } }; + expect(receipt.schema).toBe("paperclip.native-harness-web-evidence.v1"); + const [archive] = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, receipt.controller.archive.id)); + const target = join(f.mount, "isolated-web-evidence"); + expect(await restoreDurableNativeHarnessBackup({ db, storage: f.storage, companyId: f.input.companyId, + scopeDigest: archive!.scopeDigest, backupRoot: target })).toBe(true); + expect(await readFile(join(target, "current/control-plane-state.json"), "utf8")).toBe(f.raw); + expect(await readFile(join(target, "current/supervisor.json"), "utf8")).toBe(f.supervisor); + const original = join(f.sourceRoot, "codex-home/skills/retained/SKILL.md"), restored = join(target, "current/web-home/codex-home/skills/retained/SKILL.md"); + expect(await readFile(restored)).toEqual(await readFile(original)); + expect((await lstat(restored)).mode & 0o777).toBe(0o444); + expect((await readFile(restored)).length).toBe(281); + expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.id, versionId))).toEqual([]); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.input.scopeDigest))).toEqual([]); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId))).toEqual(beforeLeases); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.input.companyId))).toEqual(beforeRuns.map(r => ({ ...r, + runnerProfileJson: { ...r.runnerProfileJson, nativeHarnessFilesystemIndependence: run!.runnerProfileJson!.nativeHarnessFilesystemIndependence } }))); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-web")); vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-web/state")); + vi.stubEnv("RAILWAY_VOLUME_ID", ""); vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("replacement must not read storage")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(result.receiptDigest).toMatch(/^[a-f0-9]{64}$/); + }); + + async function webFixture(checkpointParts?: number) { + const f = await controllerFixture(true, "unversioned", checkpointParts); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const metadata = { ...lease!.metadata }; delete metadata.remoteExecutionTermination; + await db.update(environmentLeases).set({ status: "retained", cleanupStatus: "failed", releasedAt: null, metadata }) + .where(eq(environmentLeases.id, f.leaseId)); + return { ...f, input: { ...f.input, mode: "web_evidence" as const } }; + } + + it("preserves a genuinely absent controller with its supervisor and 267 durable checkpoint parts", async () => { + const f = await webFixture(267); + await rm(f.controllerPath); + const before = await db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)); + expect(before).toHaveLength(267); + const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const lastRunId = randomUUID(), execution = parseNativeExecutionInput({ ...f.execution, binding: { ...f.execution.binding, runId: lastRunId } }); + await db.insert(heartbeatRuns).values({ ...source!, id: lastRunId, status: "failed", processPid: null, processStartedAt: null, + createdAt: new Date(source!.createdAt.getTime() + 1_000), runnerProfileJson: { nativeExecutionInput: execution, retainedWarning: "existing failed admission" } }); + await db.insert(nativeRunFinalizations).values({ companyId: f.input.companyId, issueId: f.issueId, runId: lastRunId, + phase: "terminal_failure", recoveryState: "blocked" }); + await db.update(agentTaskSessions).set({ lastRunId }).where(eq(agentTaskSessions.id, f.input.selectedSessionId)); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, lastRunId)); + const receipt = run!.runnerProfileJson!.nativeHarnessFilesystemIndependence as { web: { controllerAbsent: boolean }; + controller: { archive: { id: string; controllerDigest: null } } }; + expect(receipt.web.controllerAbsent).toBe(true); + expect(receipt.controller.archive.controllerDigest).toBeNull(); + const [archive] = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, receipt.controller.archive.id)); + const target = join(f.mount, "isolated-supervisor-only"); + expect(await restoreDurableNativeHarnessBackup({ db, storage: f.storage, companyId: f.input.companyId, + scopeDigest: archive!.scopeDigest, backupRoot: target })).toBe(true); + await expect(lstat(join(target, "current/control-plane-state.json"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(join(target, "current/supervisor.json"), "utf8")).toBe(f.supervisor); + expect(await db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual(before); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-supervisor-host")); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-supervisor-host/state")); vi.stubEnv("RAILWAY_VOLUME_ID", ""); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + }); + + it("keeps WEB coverage valid through external cleanup observations without changing recovery authority", async () => { + const f = await webFixture(); + const result = await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + await db.update(environmentLeases).set({ status: "expired", cleanupStatus: "pending", expiresAt: new Date(), + lastUsedAt: new Date(), releasedAt: new Date(), metadata: { ...lease!.metadata, sandboxState: "archived", + pendingCleanupRetryAfterMs: Date.now(), pendingCleanupManualAttemptId: randomUUID(), + pendingCleanupController: { attemptId: randomUUID(), countedAttempt: true }, remoteExecutionTermination: { observation: "not a stop receipt" } } }) + .where(eq(environmentLeases.id, f.leaseId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect((await preserveNativeHarnessFilesystemIndependence(db, f.input)).receiptDigest).toBe(result.receiptDigest); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + await db.update(environmentLeases).set({ providerLeaseId: "different-allocation" }).where(eq(environmentLeases.id, f.leaseId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + }); + + it.each(["active-web", "owned-finalizer", "physical-writer", "foreign-supervisor", "missing-manifest", "symlink", "foreign-controller"])( + "refuses WEB evidence with %s and retains the original recovery state", async control => { + const f = await webFixture(), where = eq(heartbeatRuns.id, f.runId); + if (control === "active-web") await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }).where(where); + if (control === "owned-finalizer") await db.update(nativeRunFinalizations).set({ leaseOwner: "expired-but-still-owned", leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + if (control === "physical-writer") { const [run] = await db.select().from(heartbeatRuns).where(where); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run!.runnerProfileJson, nativeWorkspaceFinalizationOwner: { token: "owned" } } }).where(where); } + if (control === "foreign-supervisor") { const saved = JSON.parse(f.supervisor); saved.snapshot.identity.runId = randomUUID(); + await writeFile(f.supervisorPath, JSON.stringify(saved)); } + if (control === "foreign-controller") { const saved = JSON.parse(f.raw); saved.identity.runId = randomUUID(); await writeFile(f.controllerPath, JSON.stringify(saved)); } + if (control === "missing-manifest") { if (!("runtimeContext" in f.execution)) throw new Error("fixture runtime context missing"); + await rm(join(f.mount, "instances/default/runtime-context-assets/manifests", `${f.execution.runtimeContext.skills[0]!.bundle.digest}.json`)); } + if (control === "symlink") { const file = join(f.sourceRoot, "codex-home/history.jsonl"); await symlink(f.controllerPath, file); } + try { + const before = await db.select().from(heartbeatRuns).where(where); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect(await db.select().from(heartbeatRuns).where(where)).toEqual(before); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + } finally { if (control === "symlink") await rm(join(f.sourceRoot, "codex-home/history.jsonl")); } + }); + + it.each(["changed-byte", "changed-mode", "added-file", "controller-appeared", "changed-source"])( + "does not publish WEB evidence after %s changes during encrypted upload", async control => { + const f = await webFixture(); + const originalMode = (await lstat(f.supervisorPath)).mode & 0o777; + if (control === "controller-appeared") await rm(f.controllerPath); + const put = f.storage.putFile.bind(f.storage); let changed = false; + vi.spyOn(f.storage, "putFile").mockImplementation(async (...args) => { + const result = await put(...args); + if (!changed) { changed = true; + if (control === "changed-byte") await writeFile(f.supervisorPath, `${f.supervisor} `); + if (control === "changed-mode") await chmod(f.supervisorPath, originalMode === 0o400 ? 0o600 : 0o400); + if (control === "added-file") await writeFile(join(f.sourceRoot, "codex-home/history.jsonl"), "new uncovered bytes"); + if (control === "controller-appeared") await writeFile(f.controllerPath, f.raw); + if (control === "changed-source") { const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run!.runnerProfileJson, retainedWarning: "new source binding" } }).where(eq(heartbeatRuns.id, f.runId)); } + } return result; + }); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect(changed).toBe(true); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + }); + + it.each([false, true])("retains the authenticated WEB publication winner after a late upload (invalid: %s)", async invalid => { + const f = await webFixture(); + let release!: () => void, entered!: () => void, delay = true; + const blocked = new Promise(resolve => { release = resolve; }), ready = new Promise(resolve => { entered = resolve; }); + const get = f.storage.getObject.bind(f.storage); + vi.spyOn(f.storage, "getObject").mockImplementation(async (...args) => { + if (delay) { delay = false; entered(); await blocked; } return get(...args); + }); + const late = preserveNativeHarnessFilesystemIndependence(db, f.input); + try { + await ready; + const winner = await preserveNativeHarnessFilesystemIndependence(db, f.input); + let [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + if (invalid) { + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...published!.runnerProfileJson, + nativeHarnessFilesystemIndependence: { ...(published!.runnerProfileJson!.nativeHarnessFilesystemIndependence as Record), assetsDigest: "0".repeat(64) } } }) + .where(eq(heartbeatRuns.id, f.runId)); + [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + } + release(); + if (invalid) await expect(late).rejects.toThrow("unverified"); + else expect((await late).receiptDigest).toBe(winner.receiptDigest); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(published); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + } finally { release(); await late.catch(() => undefined); } + }); + + it.each(["missing-entry", "changed-source"])("refuses existing WEB coverage after %s without replacing its receipt", async control => { + const f = await webFixture(); await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = run!.runnerProfileJson!.nativeHarnessFilesystemIndependence as Record & { web: { entries: Array<{ path: string }> } }; + const profile = control === "missing-entry" ? { ...run!.runnerProfileJson, nativeHarnessFilesystemIndependence: { ...receipt, + web: { ...receipt.web, entries: receipt.web.entries.filter(entry => entry.path !== "web-supervisor.json") } } } + : { ...run!.runnerProfileJson, sessionCheckpoint: { changed: true } }; + await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson).toEqual(profile); + }); + async function historyFixture(keepHistoricalHeaders = false, includeRunnerState = false) { const f = await controllerFixture(true); const get = f.storage.getObject.bind(f.storage); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.ts index ebed5d1f0e8..f8487b79922 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.ts @@ -1,4 +1,5 @@ import { constants } from "node:fs"; +import type { Stats } from "node:fs"; import { lstat, open, opendir, readdir, realpath } from "node:fs/promises"; import { createHash } from "node:crypto"; import { basename, join, posix } from "node:path"; @@ -6,6 +7,7 @@ import { and, asc, eq, inArray, or, sql } from "drizzle-orm"; import { agentTaskSessions, companySkillRepositorySnapshots, companySkillVersions, environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunnerCheckpoints, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { StorageService } from "../../storage/types.js"; +import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import { adapterExecutionControls } from "../adapter-execution-control.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "../remote-execution-termination.js"; @@ -19,14 +21,15 @@ import { captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, p import { nativeSha256 } from "./canonical.js"; import { readRuntimeSkillPreservation } from "./runtime-skill-preservation-reference.js"; import { findNativeSessionResumeRun } from "./native-session-resume.js"; -import { resolveNativeHarnessPersistenceProfile, scopedRunnerdStateRoot } from "./native-session-executor.js"; +import { nativeControllerMigrationDescriptor, resolveNativeHarnessPersistenceProfile, scopedRunnerdStateRoot } from "./native-session-executor.js"; import { readNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; import { tryLockNativeWorkspaceFinalization } from "./native-workspace-finalization-ownership.js"; import { assertOriginalHarnessVolumeSource, type OriginalHarnessVolumeCapability } from "./native-harness-original-volume.js"; import { isKnownNativeSessionRootEntry } from "./native-session-root-entries.js"; import { readLegacyNativeControllerArchiveSource, readLegacyNativeControllerArchiveDigests } from "./native-controller-migration.js"; import { archiveNativeControllerSourceEvidence, legacyNativeControllerArchive, nativeControllerEvidenceDescriptor, - nativeControllerHistoryManifest, type NativeControllerHistoryEntry, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; + nativeControllerHistoryManifest, nativeControllerWebManifest, type NativeControllerWebEntry, + type NativeControllerHistoryEntry, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; import { nativeControllerArchiveCoordinatorDigest, nativeControllerArchiveProfileDigest, type SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; import { loadNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; import { readNativeControllerRecovery } from "./native-controller-recovery.js"; @@ -34,6 +37,7 @@ import { verifyDurableNativeHarnessBackup } from "./native-harness-backup-storag const KEY = "nativeHarnessFilesystemIndependence"; const SCHEMA = "paperclip.native-harness-filesystem-independence.v1"; +const WEB_SCHEMA = "paperclip.native-harness-web-evidence.v1"; const terminal = ["succeeded", "failed", "cancelled", "interrupted", "timed_out"]; const sha = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); const digest = (value: unknown) => nativeSha256(JSON.parse(JSON.stringify(value))); @@ -44,17 +48,30 @@ type VersionBinding = { id: string; digest: string }; type HistoryEpoch = { name: string; runId: string; identity: DurableRecoveryIdentity; identityDigest: string; headerDigest: string | null }; type HistoryBinding = { entries: NativeControllerHistoryManifestEntry[]; epochs: HistoryEpoch[] }; type CapturedHistory = { entries: NativeControllerHistoryEntry[]; binding: HistoryBinding; localDigest: string }; +type WebSkill = { runId: string; key: string; runtimeName: string; versionId: string | null; bundleDigest: string; + manifestDigest: string; fileCount: number; totalBytes: number }; +type WebBinding = { entries: NativeControllerHistoryManifestEntry[]; skills: WebSkill[]; rootPresent: boolean; + profileDigest: string; controllerAbsent: boolean; excluded: Array<{ path: string; type: "directory" | "file"; mode: number; size: number }> }; +type CapturedWeb = { entries: NativeControllerWebEntry[]; binding: WebBinding; localDigest: string }; type ControllerBinding = { checkpointPartsDigest: string; history?: HistoryBinding; archive: null | { id: string; digest: string; - controllerDigest: string; supervisorDigest: string | null } }; -type Receipt = { schema: typeof SCHEMA; selectedSessionId: string; lastRunId: string; sourceRunId: string; scopeDigest: string; + controllerDigest: string | null; supervisorDigest: string | null } }; +type Receipt = { schema: typeof SCHEMA | typeof WEB_SCHEMA; selectedSessionId: string; lastRunId: string; sourceRunId: string; scopeDigest: string; factsDigest: string; sourceCapabilityFingerprint: string; localFactsDigest: string; assetsDigest: string; - packages: PackageBinding[]; versions: VersionBinding[]; controller: ControllerBinding; observedAt: string }; + packages: PackageBinding[]; versions: VersionBinding[]; controller: ControllerBinding; observedAt: string; web?: WebBinding }; type Scope = { companyId: string; selectedSessionId: string; scopeDigest: string }; type Run = typeof heartbeatRuns.$inferSelect; const evidenceOwners = new WeakMap(); + history?: CapturedHistory; web?: CapturedWeb; archiveManifestSha256: string; signal: AbortSignal }>(); -async function databaseFacts(db: Db, input: Scope, lock: boolean) { +function webLeaseBinding(lease: typeof environmentLeases.$inferSelect) { + const fields = Object.fromEntries(Object.entries(lease).filter(([key]) => !["status", "releasedAt", "updatedAt", "lastUsedAt", "expiresAt", "failureReason", "cleanupStatus"].includes(key))); + const metadata = Object.fromEntries(Object.entries(lease.metadata ?? {}).filter(([key]) => !["pendingCleanupAttemptId", "pendingCleanupInFlight", + "pendingCleanupLeaseExpiresAtMs", "pendingCleanupRetryAfterMs", "pendingCleanupRetryAttempts", "pendingCleanupRetryCapWarned", + "pendingCleanupController", "pendingCleanupManualAttemptId", "remoteExecutionTermination", "sandboxStopAndRetainReceipt", "sandboxState", "resumedFromState"].includes(key))); + return { ...fields, metadata }; +} + +async function databaseFacts(db: Db, input: Scope, lock: boolean, webOnly = false) { const [selection] = await db.select().from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, input.companyId), eq(agentTaskSessions.id, input.selectedSessionId), eq(agentTaskSessions.adapterType, "paperclip_runner"))); if (!selection?.lastRunId || !/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail(); @@ -113,10 +130,12 @@ async function databaseFacts(db: Db, input: Scope, lock: boolean) { let matchingStamp = false; const terminations = []; for (const lease of leases) { - const resolved = await resolveRemoteTerminationLease(db, lease); - const receipt = resolved.metadata?.remoteExecutionTermination as { state?: unknown } | undefined; - if (!hasRemoteTerminationReceipt(resolved) || receipt?.state !== "destroyed") fail(); - terminations.push({ leaseId: lease.id, receipt }); + if (!webOnly) { + const resolved = await resolveRemoteTerminationLease(db, lease); + const receipt = resolved.metadata?.remoteExecutionTermination as { state?: unknown } | undefined; + if (!hasRemoteTerminationReceipt(resolved) || receipt?.state !== "destroyed") fail(); + terminations.push({ leaseId: lease.id, receipt }); + } const stamp = readNativeHarnessBackupStamp(lease.metadata?.nativeHarnessBackup, lease.providerLeaseId ?? ""); if (stamp?.sessionScopeSha256 === input.scopeDigest && stamp.normalizedSessionId === anchor.nativeSessionId && runs.some(run => run.runnerInstanceId === stamp.runnerInstanceId) @@ -124,7 +143,8 @@ async function databaseFacts(db: Db, input: Scope, lock: boolean) { } if (!matchingStamp) fail(); const factsDigest = digest({ selection, runs: runs.map(run => ({ ...run, runnerProfileJson: profile(run.runnerProfileJson) })), owners, - task: { id: task.id, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId }, leases, terminations, + task: { id: task.id, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId }, + leases: webOnly ? leases.map(webLeaseBinding) : leases, terminations, sourceRunId: source.id, sourceProfile: source.runnerProfileJson }); return { anchor, execution, source, factsDigest, runs }; } @@ -169,35 +189,46 @@ async function assetBindings(db: Db, anchor: Run, sourceRunId: string) { } function matchingReceipt(value: unknown, facts: Awaited>, input: Scope, - bindings: Awaited>["bindings"], controller: ControllerBinding): Receipt | null { + bindings: Awaited>["bindings"], controller: ControllerBinding, web?: WebBinding): Receipt | null { const receipt = value as Receipt | undefined; - if (receipt?.schema !== SCHEMA || receipt.selectedSessionId !== input.selectedSessionId || receipt.scopeDigest !== input.scopeDigest + if (receipt?.schema !== (web ? WEB_SCHEMA : SCHEMA) || receipt.selectedSessionId !== input.selectedSessionId || receipt.scopeDigest !== input.scopeDigest || receipt.lastRunId !== facts.anchor.id || receipt.sourceRunId !== facts.source.id || receipt.factsDigest !== facts.factsDigest || !/^[a-f0-9]{64}$/.test(receipt.sourceCapabilityFingerprint) || !/^[a-f0-9]{64}$/.test(receipt.localFactsDigest) || !Number.isFinite(Date.parse(receipt.observedAt)) || !Array.isArray(receipt.packages) || !Array.isArray(receipt.versions)) return null; return digest(bindings) === receipt.assetsDigest && digest(receipt.packages) === digest(bindings.packages) && digest(receipt.versions) === digest(bindings.versions) && receipt.controller != null - && digest(receipt.controller) === digest(controller) ? receipt : null; + && digest(receipt.controller) === digest(controller) && digest(receipt.web ?? null) === digest(web ?? null) ? receipt : null; } export async function hasValidNativeHarnessFilesystemIndependence(db: Db, input: Scope): Promise { try { - const facts = await databaseFacts(db, input, false); - const assets = await assetBindings(db, facts.anchor, facts.source.id); + const [anchor] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(agentTaskSessions) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, agentTaskSessions.lastRunId), eq(heartbeatRuns.companyId, agentTaskSessions.companyId), + eq(heartbeatRuns.agentId, agentTaskSessions.agentId))) + .where(and(eq(agentTaskSessions.id, input.selectedSessionId), eq(agentTaskSessions.companyId, input.companyId))); + const value = anchor?.profile?.[KEY] as Receipt | undefined; + const webOnly = value?.schema === WEB_SCHEMA; + const facts = await databaseFacts(db, input, false, webOnly); const existing = facts.anchor.runnerProfileJson?.[KEY] as Receipt | undefined; - const controller = await controllerBinding(db, facts, existing?.controller?.archive ?? null, false, existing?.controller?.history); - return matchingReceipt(existing, facts, input, assets.bindings, controller) !== null; + if (webOnly && (!existing?.web || !validWebBinding(facts, existing.web))) return false; + const bindings = webOnly ? { packages: [], versions: [] } : (await assetBindings(db, facts.anchor, facts.source.id)).bindings; + const controller = await controllerBinding(db, facts, existing?.controller?.archive ?? null, false, existing?.controller?.history, existing?.web); + return matchingReceipt(existing, facts, input, bindings, controller, existing?.web) !== null; } catch { return false; } } export async function assertNativeHarnessFilesystemArchiveSource(db: Db, owner: SettledControllerArchiveOwner) { const registered = evidenceOwners.get(owner); if (!registered || !owner.filesystemIndependence) fail(); - const facts = await databaseFacts(db, registered.scope, true); + const facts = await databaseFacts(db, registered.scope, true, registered.web !== undefined); if (facts.factsDigest !== registered.factsDigest || facts.factsDigest !== owner.filesystemIndependence.factsDigest || facts.source.id !== owner.runId || facts.anchor.id !== owner.selectedLastRunId) fail(); const history = await captureAuthorityHistory(db, facts, registered.signal); if (history?.localDigest !== registered.history?.localDigest || digest(history?.binding ?? null) !== digest(registered.history?.binding ?? null)) fail(); + if (registered.web) { + const web = await captureWebEvidence(facts, registered.signal); + if (web.localDigest !== registered.web.localDigest || digest(web.binding) !== digest(registered.web.binding)) fail(); + } } export function assertNativeHarnessFilesystemArchiveContents(owner: SettledControllerArchiveOwner, archive: { manifestSha256: string }) { @@ -313,12 +344,15 @@ async function captureAuthorityHistory(db: Db, facts: Awaited>, archive: ControllerBinding["archive"], lock: boolean, - history?: HistoryBinding): Promise { + history?: HistoryBinding, web?: WebBinding): Promise { const partsQuery = db.select().from(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.companyId, facts.anchor.companyId), eq(nativeRunnerCheckpoints.issueId, facts.anchor.nativeIssueId!), eq(nativeRunnerCheckpoints.runId, facts.source.id))) .orderBy(asc(nativeRunnerCheckpoints.part)); const parts = await (lock ? partsQuery.for("share") : partsQuery); if (archive) { + if (web && (!validWebBinding(facts, web) || web.controllerAbsent !== (archive.controllerDigest === null))) fail(); + if (web && (web.entries.find(entry => entry.path === "web-supervisor.json" && entry.type === "file") as + Extract | undefined)?.sha256 !== (archive.supervisorDigest ?? undefined)) fail(); if (!parts.some(row => row.part === "header") || !readNativeControllerRecovery( facts.source.runnerProfileJson.nativeControllerRecovery, facts.source.runnerProfileJson.sessionCheckpoint)) fail(); const rowQuery = db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, facts.anchor.companyId), @@ -339,9 +373,10 @@ async function controllerBinding(db: Db, facts: Awaited>, input: Scope & { source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; -}, history?: CapturedHistory): Promise { +}, history?: CapturedHistory, web?: CapturedWeb): Promise { const material = await readLegacyNativeControllerArchiveSource(facts.source); if (material.paths.root !== scopedRunnerdStateRoot(facts.execution)) fail(); - if (material.controller === undefined) { + if (material.controller === undefined && !web) { if (material.supervisor !== undefined || history) fail(); return controllerBinding(db, facts, null, false); } - const identity = JSON.parse(material.controller).identity as Record | undefined; - if (identity?.runId !== facts.source.id || identity.runnerInstanceId !== facts.source.runnerInstanceId - || identity.normalizedSessionId !== facts.anchor.nativeSessionId) fail(); const checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: facts.anchor.nativeIssueId!, runId: facts.source.id }); if (!checkpoint || !readNativeControllerRecovery(facts.source.runnerProfileJson.nativeControllerRecovery, facts.source.runnerProfileJson.sessionCheckpoint)) fail(); - const captured = captureLegacyDurablePrpCheckpoint(material.controller, { runId: facts.source.id, - runnerInstanceId: facts.source.runnerInstanceId!, normalizedSessionId: facts.anchor.nativeSessionId! }, - { version: checkpoint.runnerVersion, digest: checkpoint.runnerDigest }); - if (captured.records.size !== checkpoint.records.size || [...captured.records].some(([key, value]) => { - const durable = checkpoint.records.get(key); return durable === undefined || digest(JSON.parse(value)) !== digest(JSON.parse(durable)); - })) fail(); + if (web && material.supervisor !== undefined) { + const saved = JSON.parse(material.supervisor), identity = saved.snapshot?.identity; + if (saved.schema !== "paperclip.native-session-supervisor.v1" || identity?.companyId !== facts.anchor.companyId + || identity.agentId !== facts.anchor.agentId || identity.issueId !== facts.anchor.nativeIssueId + || identity.sessionId !== facts.anchor.nativeSessionId || !facts.runs.some(run => run.id === identity.runId)) fail(); + } + if (material.controller !== undefined) { + const identity = JSON.parse(material.controller).identity as Record | undefined; + if (identity?.runId !== facts.source.id || identity.runnerInstanceId !== facts.source.runnerInstanceId + || identity.normalizedSessionId !== facts.anchor.nativeSessionId) fail(); + const captured = captureLegacyDurablePrpCheckpoint(material.controller, { runId: facts.source.id, + runnerInstanceId: facts.source.runnerInstanceId!, normalizedSessionId: facts.anchor.nativeSessionId! }, + { version: checkpoint.runnerVersion, digest: checkpoint.runnerDigest }); + if (captured.records.size !== checkpoint.records.size || [...captured.records].some(([key, value]) => { + const durable = checkpoint.records.get(key); return durable === undefined || digest(JSON.parse(value)) !== digest(JSON.parse(durable)); + })) fail(); + } const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, facts.source.id))); if (!coordinator) fail(); @@ -375,14 +418,15 @@ async function preserveControllerEvidence(db: Db, facts: Awaited>): WebSkill[] { + const rows = facts.source.id === facts.anchor.id ? [facts.anchor] : [facts.anchor, facts.source]; + return rows.flatMap(row => { + const input = parseNativeExecutionInput(row.runnerProfileJson?.nativeExecutionInput); + if (!("runtimeContext" in input) || input.binding.companyId !== row.companyId || input.binding.agentId !== row.agentId + || input.binding.issueId !== row.nativeIssueId || input.binding.runId !== row.id + || input.session.normalizedSessionId !== row.nativeSessionId) fail(); + return input.runtimeContext.skills.map(skill => ({ runId: row.id, key: skill.key, runtimeName: skill.runtimeName, + versionId: skill.versionId, bundleDigest: skill.bundle.digest, manifestDigest: skill.bundle.manifestDigest, + fileCount: skill.bundle.fileCount, totalBytes: skill.bundle.totalBytes })); + }).sort((a, b) => `${a.runId}:${a.key}:${a.runtimeName}`.localeCompare(`${b.runId}:${b.key}:${b.runtimeName}`)); +} + +function webProfile(facts: Awaited>) { + return resolveNativeHarnessPersistenceProfile(facts.execution).directories + .map(directory => ({ name: directory.name, location: directory.location, excludeEntries: directory.excludeEntries })); +} + +function validWebBinding(facts: Awaited>, web: WebBinding): boolean { + try { + if (!web || typeof web.rootPresent !== "boolean" || typeof web.controllerAbsent !== "boolean" || !Array.isArray(web.entries) + || !Array.isArray(web.skills) || !Array.isArray(web.excluded) || web.entries.length > 128 || web.excluded.length > 128 + || web.profileDigest !== digest(webProfile(facts)) || digest(web.skills) !== digest(webSkills(facts))) return false; + const profiles = webProfile(facts), manifests = new Map(web.skills.map(skill => [skill.bundleDigest, skill.manifestDigest])); + for (const [bundle, manifest] of manifests) if (!web.entries.some(entry => entry.path === `web-manifests/${bundle}.json` + && entry.type === "file" && entry.sha256 === manifest)) return false; + if (web.entries.some(entry => { + const [prefix, name, ...rest] = entry.path.split("/"); + if (prefix === "web-supervisor.json") return name !== undefined || entry.type !== "file"; + if (prefix === "web-manifests") return name === undefined ? entry.type !== "directory" + : !/^[a-f0-9]{64}\.json$/.test(name) || rest.length > 0 || entry.type !== "file" || manifests.get(name.slice(0, -5)) !== entry.sha256; + if (!["web-home", "web-state"].includes(prefix!)) return true; + if (!web.rootPresent && name !== undefined) return true; + if (name === undefined) return entry.type !== "directory"; + if (prefix === "web-state") return !isKnownNativeSessionRootEntry(name) || profiles.some(profile => profile.name === name); + const profile = profiles.find(profile => profile.name === name), rel = rest.join("/"); + return !profile || profile.excludeEntries.some(excluded => rel === excluded || rel.startsWith(`${excluded}/`)); + })) return false; + return web.excluded.every(entry => { + const [name, ...rest] = entry.path.split("/"), rel = rest.join("/"), profile = profiles.find(profile => profile.name === name); + assertSkillSnapshotPath(entry.path); + return profile?.excludeEntries.some(excluded => rel === excluded || rel.startsWith(`${excluded}/`)) + && ["directory", "file"].includes(entry.type) && Number.isInteger(entry.mode) && (entry.mode & ~0o777) === 0 + && Number.isSafeInteger(entry.size) && entry.size >= 0; + }); + } catch { return false; } +} + +async function captureWebEvidence(facts: Awaited>, signal: AbortSignal): Promise { + const root = scopedRunnerdStateRoot(facts.execution), profiles = webProfile(facts), entries: NativeControllerWebEntry[] = []; + const metadata: unknown[] = [], excluded: WebBinding["excluded"] = [], inodes = new Set(); + let bytes = 0, count = 0; + const directoryNames = async (path: string) => { + const names: string[] = []; + for await (const entry of await opendir(path)) { + signal.throwIfAborted(); if (names.length >= 128) fail(); names.push(entry.name); + } + return names.sort(); + }; + const stat = (path: string) => lstat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + const rootStat = await stat(root), instance = await lstat(resolvePaperclipInstanceRoot()); + const read = async (file: string, s: Stats) => { + const key = `${s.dev}:${s.ino}`; + if (!s.isFile() || s.nlink !== 1 || inodes.has(key) || s.dev !== instance.dev || s.uid !== instance.uid + || (s.mode & 0o7000) !== 0 || await realpath(file) !== file + || s.size > 16 * 1024 * 1024 || (bytes += s.size) > 64 * 1024 * 1024) fail(); + inodes.add(key); + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const opened = await handle.stat(); + if (!opened.isFile() || opened.dev !== s.dev || opened.ino !== s.ino || opened.size !== s.size) fail(); + const buffer = Buffer.alloc(s.size + 1); let length = 0; + while (length < buffer.length) { signal.throwIfAborted(); const result = await handle.read(buffer, length, buffer.length - length, length); + if (!result.bytesRead) break; length += result.bytesRead; } + const after = await handle.stat(), named = await lstat(file); + if (length !== s.size || after.size !== s.size || after.mtimeMs !== s.mtimeMs || after.ctimeMs !== s.ctimeMs + || named.dev !== s.dev || named.ino !== s.ino) fail(); + metadata.push([file, s.dev, s.ino, s.size, s.mode, s.mtimeMs, s.ctimeMs]); + return buffer.subarray(0, length); + } finally { await handle.close(); } + }; + const visit = async (file: string, path: string, relative: string): Promise => { + signal.throwIfAborted(); if (++count > 128) fail(); assertSkillSnapshotPath(path); + const s = await lstat(file); + if (s.isSymbolicLink() || !s.isDirectory() && !s.isFile() || s.dev !== instance.dev || s.uid !== instance.uid || (s.mode & 0o7000) !== 0) fail(); + metadata.push([path, s.dev, s.ino, s.size, s.mode, s.mtimeMs, s.ctimeMs]); + const [name, ...rest] = relative.split("/"), rel = rest.join("/"), profile = profiles.find(profile => profile.name === name); + if (profile?.excludeEntries.some(exclusion => rel === exclusion || rel.startsWith(`${exclusion}/`))) { + excluded.push({ path: relative, type: s.isDirectory() ? "directory" : "file", mode: s.mode & 0o777, size: s.isFile() ? s.size : 0 }); return; + } + if (s.isFile()) { entries.push({ path, type: "file", mode: s.mode & 0o777, content: await read(file, s) }); return; } + entries.push({ path, type: "directory", mode: s.mode & 0o777 }); + const names = await directoryNames(file); + for (const name of names) await visit(join(file, name), `${path}/${name}`, `${relative}/${name}`); + const after = await lstat(file); + if (after.dev !== s.dev || after.ino !== s.ino || after.ctimeMs !== s.ctimeMs || digest(await directoryNames(file)) !== digest(names)) fail(); + }; + if (rootStat) { + if (!rootStat.isDirectory() || rootStat.isSymbolicLink() || await realpath(root) !== root || rootStat.dev !== instance.dev || rootStat.uid !== instance.uid) fail(); + metadata.push([root, rootStat.dev, rootStat.ino, rootStat.mode, rootStat.mtimeMs, rootStat.ctimeMs]); + const names = await directoryNames(root); + if (names.some(name => !isKnownNativeSessionRootEntry(name))) fail(); + for (const name of ["continuity-breaks", "provider-artifacts", "failover-backups"]) { + const s = await stat(join(root, name)); + if (s && (!s.isDirectory() || s.isSymbolicLink() || (await directoryNames(join(root, name))).length)) fail(); + } + entries.push({ path: "web-home", type: "directory", mode: rootStat.mode & 0o777 }, + { path: "web-state", type: "directory", mode: rootStat.mode & 0o777 }); + for (const name of names) { + if (["runner", "codex-home", "opencode", "acpx"].includes(name) && !profiles.some(profile => profile.name === name)) fail(); + await visit(join(root, name), `${profiles.some(profile => profile.name === name) ? "web-home" : "web-state"}/${name}`, name); + } + if (digest(await directoryNames(root)) !== digest(names)) fail(); + } + const sourceExecution = parseNativeExecutionInput(facts.source.runnerProfileJson.nativeExecutionInput); + const supervisor = nativeControllerMigrationDescriptor(sourceExecution, facts.source.runnerProfileJson.nativeToolContractFingerprint).supervisorPath; + const supervisorStat = sourceExecution.session.lifecyclePolicy.mode === "warm" ? await stat(supervisor) : null; + if (supervisorStat) { + const content = await read(supervisor, supervisorStat); + entries.push({ path: "web-supervisor.json", type: "file", mode: supervisorStat.mode & 0o777, content }); + } else metadata.push(["web-supervisor.json", null]); + const skills = webSkills(facts); + if (skills.length) entries.push({ path: "web-manifests", type: "directory", mode: 0o700 }); + const seen = new Set(); + for (const skill of skills) { + if (seen.has(skill.bundleDigest)) continue; seen.add(skill.bundleDigest); + if (!/^[a-f0-9]{64}$/.test(skill.bundleDigest)) fail(); + const path = join(resolvePaperclipInstanceRoot(), "runtime-context-assets/manifests", `${skill.bundleDigest}.json`), s = await lstat(path); + const content = await read(path, s); + runtimeFiles({ bundle: { digest: skill.bundleDigest, manifestDigest: skill.manifestDigest, fileCount: skill.fileCount, totalBytes: skill.totalBytes } }, content.toString("utf8")); + entries.push({ path: `web-manifests/${skill.bundleDigest}.json`, type: "file", mode: s.mode & 0o777, content }); + } + if (entries.length > 128) fail(); + const material = await readLegacyNativeControllerArchiveDigests(facts.source); + const binding: WebBinding = { entries: nativeControllerWebManifest(entries), skills, rootPresent: rootStat !== null, + profileDigest: digest(profiles), controllerAbsent: material.controllerDigest === undefined, excluded }; + if (!validWebBinding(facts, binding)) fail(); + return { entries, binding, localDigest: digest(metadata) }; +} + async function localFacts(facts: Awaited>, assets: Awaited>, storage: StorageService, db: Db, verifyStorage: boolean, signal: AbortSignal, controller: ControllerBinding, plans = new Map()) { const root = scopedRunnerdStateRoot(facts.execution), metadata: unknown[] = [], store = skillPackageStore(db, storage); @@ -516,11 +700,55 @@ async function localFacts(facts: Awaited>, asse return { digest: digest(metadata), plans }; } -export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: Scope & { +async function preserveWebEvidence(db: Db, input: Scope & { source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; +}) { + const facts = await databaseFacts(db, input, false, true), history = await captureAuthorityHistory(db, facts, input.signal); + const web = await captureWebEvidence(facts, input.signal); + const controller = await preserveControllerEvidence(db, facts, input, history, web), bindings = { packages: [], versions: [] }; + const previous = facts.anchor.runnerProfileJson?.[KEY]; + if (previous != null) { + if (!await hasValidNativeHarnessFilesystemIndependence(db, input)) fail(); + return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(previous) }; + } + const receipt: Receipt = { schema: WEB_SCHEMA, selectedSessionId: input.selectedSessionId, scopeDigest: input.scopeDigest, + lastRunId: facts.anchor.id, sourceRunId: facts.source.id, factsDigest: facts.factsDigest, + sourceCapabilityFingerprint: input.source.fingerprint, localFactsDigest: web.localDigest, + assetsDigest: digest(bindings), ...bindings, controller, web: web.binding, observedAt: new Date().toISOString() }; + let published = receipt; + const publications: ActivityPublication[] = []; + await db.transaction(async transaction => { + const tx = transaction as unknown as Db; + await tx.execute(sql`set local lock_timeout = '1s'`); + const current = await databaseFacts(tx, input, true, true); + if (current.factsDigest !== facts.factsDigest) fail(); + const currentController = await controllerBinding(tx, current, controller.archive, true, controller.history, web.binding); + const currentWeb = await captureWebEvidence(current, input.signal); + if (digest(currentController) !== digest(controller) || currentWeb.localDigest !== web.localDigest + || digest(currentWeb.binding) !== digest(web.binding)) fail(); + await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); + const existing = current.anchor.runnerProfileJson?.[KEY]; + if (existing != null) { + const winner = matchingReceipt(existing, current, input, bindings, currentController, web.binding); + if (!winner) fail(); published = winner; return; + } + await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...current.anchor.runnerProfileJson, [KEY]: receipt } }) + .where(and(eq(heartbeatRuns.id, current.anchor.id), eq(heartbeatRuns.companyId, input.companyId))); + await logActivity(tx, { companyId: input.companyId, actorType: "system", actorId: "native-harness-preservation", + action: "native_harness_filesystem_independence_verified", entityType: "heartbeat_run", entityId: current.anchor.id, + agentId: current.anchor.agentId, runId: current.anchor.id, details: { scopeDigest: input.scopeDigest, sourceRunId: current.source.id, + sourceCapabilityFingerprint: receipt.sourceCapabilityFingerprint, factsDigest: receipt.factsDigest, coverage: "web_evidence" } }, publications); + }); + for (const publication of publications) publishActivity(publication); + return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(published) }; +} + +export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: Scope & { + source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; mode?: "web_evidence"; }) { await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); if (input.storage.provider !== "s3") fail(); + if (input.mode === "web_evidence") return preserveWebEvidence(db, input); const facts = await databaseFacts(db, input, false), assets = await assetBindings(db, facts.anchor, facts.source.id); const history = await captureAuthorityHistory(db, facts, input.signal); const controller = await preserveControllerEvidence(db, facts, input, history);