diff --git a/doc/DATABASE.md b/doc/DATABASE.md index bb1594be275..fe8df430701 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -105,8 +105,10 @@ Files retain their bytes and modes, including local edits and a skill whose version row was deleted. Such a skill remains forensic evidence: the owner does not recreate its version or make it executable. Canonical credential exclusions stay excluded. Unknown directories, symlinks, incomplete manifests, active web -writers and changed source facts fail closed. The same 128-entry, 16-MiB-file and -64-MiB-total limits apply independently to this inventory. +writers and changed source facts fail closed. This WEB inventory permits at most +10,000 entries, 4 MiB of manifest metadata, and 4,096 bytes per path. The 16-MiB +per-file and 64-MiB total payload limits remain unchanged. Authority history keeps +its separate 128-entry and 32-epoch bounds. A genuinely absent raw controller stays absent. A supervisor-only archive still requires the exact durable controller checkpoint and source identity. Old diff --git a/server/src/services/native-runtime/native-controller-archive.ts b/server/src/services/native-runtime/native-controller-archive.ts index b698c49ba6e..2584c8e19b9 100644 --- a/server/src/services/native-runtime/native-controller-archive.ts +++ b/server/src/services/native-runtime/native-controller-archive.ts @@ -7,7 +7,14 @@ import type { StorageService } from "../../storage/types.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import type { SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; -import { assertSkillSnapshotPath } from "../skill-snapshot.js"; +import { assertSkillSnapshotPath, MAX_SKILL_SNAPSHOT_FILES } from "../skill-snapshot.js"; +import { MAX_NATIVE_HARNESS_PATH_BYTES } from "./native-harness-backup-archive.js"; + +export const NATIVE_CONTROLLER_WEB_LIMITS = { + entries: MAX_SKILL_SNAPSHOT_FILES, + metadataBytes: 4 * 1024 * 1024, + pathBytes: MAX_NATIVE_HARNESS_PATH_BYTES, +} as const; export type NativeControllerHistoryEntry = { path: string; mode: number; type: "directory" } | { path: string; mode: number; type: "file"; content: string }; @@ -60,18 +67,22 @@ export function nativeControllerEvidenceDescriptor(source: { companyId: string; } } } + if (source.web && source.web.entries.length > NATIVE_CONTROLLER_WEB_LIMITS.entries) { + throw new Error("native_controller_archive_authority_invalid"); + } const web = source.web ? { bindingDigest: source.web.bindingDigest, entries: source.web.entries.map(entry => entry.type === "directory" ? { path: entry.path, mode: entry.mode, type: entry.type } : { path: entry.path, mode: entry.mode, type: entry.type, size: entry.size, sha256: entry.sha256 }) } : undefined; if (source.controllerDigest === null && !web) throw new Error("native_controller_archive_authority_invalid"); if (web) { const paths = new Set(); let bytes = 0; - if (source.purpose !== "filesystem_independence" || web.entries.length > 128 || !/^[a-f0-9]{64}$/.test(web.bindingDigest)) { + if (source.purpose !== "filesystem_independence" || !/^[a-f0-9]{64}$/.test(web.bindingDigest)) { throw new Error("native_controller_archive_authority_invalid"); } for (const entry of web.entries) { assertSkillSnapshotPath(entry.path); - if (!/^(?:(web-home|web-state|web-manifests)(\/|$)|web-supervisor\.json$)/.test(entry.path) || paths.has(entry.path) + if (Buffer.byteLength(entry.path) > NATIVE_CONTROLLER_WEB_LIMITS.pathBytes + || !/^(?:(web-home|web-state|web-manifests)(\/|$)|web-supervisor\.json$)/.test(entry.path) || paths.has(entry.path) || !["directory", "file"].includes(entry.type) || !Number.isInteger(entry.mode) || entry.mode < 0 || (entry.mode & ~0o777) !== 0 || entry.path.includes("/") && !paths.has(entry.path.slice(0, entry.path.lastIndexOf("/"))) @@ -81,6 +92,9 @@ export function nativeControllerEvidenceDescriptor(source: { companyId: string; } paths.add(entry.path); } + if (Buffer.byteLength(JSON.stringify(web.entries)) > NATIVE_CONTROLLER_WEB_LIMITS.metadataBytes) { + throw new Error("native_controller_archive_authority_invalid"); + } } const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId, issueId: source.issueId, runId: source.runId, reason: source.purpose === "filesystem_independence" @@ -91,6 +105,9 @@ export function nativeControllerEvidenceDescriptor(source: { companyId: string; ...(web ? Object.fromEntries(web.entries.filter(entry => entry.type === "file").map(entry => [entry.path, entry.sha256])) : {}) }, ...(history ? { history, historyBindingDigest: source.historyBindingDigest } : {}), ...(web ? { web: { entries: web.entries, bindingDigest: web.bindingDigest, controllerAbsent: source.controllerDigest === null } } : {}) }); + if (web && Buffer.byteLength(manifest) > NATIVE_CONTROLLER_WEB_LIMITS.metadataBytes) { + throw new Error("native_controller_archive_authority_invalid"); + } return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` }; } diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts index d1640ebb7e9..13c90368082 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts @@ -32,7 +32,7 @@ import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-stora import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { nativeCheckpointRetirement } from "./native-runner-checkpoint-retirement.js"; import { nativeSha256 } from "./canonical.js"; -import { nativeControllerEvidenceDescriptor } from "./native-controller-archive.js"; +import { nativeControllerEvidenceDescriptor, NATIVE_CONTROLLER_WEB_LIMITS, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; vi.mock("../../build-commit.js", () => ({ readBuildCommit: () => "a".repeat(40) })); @@ -43,7 +43,7 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await db.delete(agentTaskSessions); for (const root of roots.splice(0)) await removeRuntimeAssetTree(root); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); - async function fixture(kind: "unversioned" | "repository" | "package" | "deleted" | "none" = "unversioned", managedWorkspace = false) { + async function fixture(kind: "unversioned" | "repository" | "package" | "deleted" | "none" | "many-web-files" = "unversioned", managedWorkspace = false) { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); const mount = await realpath(await mkdtemp(join(tmpdir(), "native-harness-original-"))); roots.push(mount); const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); @@ -71,7 +71,16 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" await db.insert(agents).values({ id: agentId, companyId, name: "Retained source" }); await db.insert(issues).values({ id: issueId, companyId, title: "Retain prior failure", status: "blocked" }); const runtimeContext = nativeRuntimeContextFixture(); - if (kind !== "none") { + if (kind === "many-web-files") { + runtimeContext.skills = []; + for (const [name, count] of [["first", 462], ["second", 462], ["third", 3]] as const) { + const bundle = await materializeAsset(Array.from({ length: count }, (_, index) => ({ + path: `file-${index}.md`, content: Buffer.from(`${name}-${index}\n`.padEnd(20_000, "x")), mode: 0o444, + }))); + runtimeContext.skills.push({ key: name, runtimeName: name, versionId: null, bundle }); + } + runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext); + } else if (kind !== "none") { let versionId: string | null = null; const files: CompanySkillVersionFileInventoryEntry[] = kind === "deleted" ? [{ path: "SKILL.md", kind: "skill", content: "# Historical acceptance skill\n\n".padEnd(281, "x") }] @@ -122,7 +131,9 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" if (kind !== "none") { if (kind === "unversioned") await preserveRuntimeSkillAssets(db, { companyId, runId }, storage); await mkdir(join(scopedRunnerdStateRoot(execution), "codex-home/skills"), { recursive: true, mode: 0o700 }); - await cp(runtimeContext.skills[0]!.bundle.rootPath, join(scopedRunnerdStateRoot(execution), "codex-home/skills/retained"), { recursive: true }); + for (const skill of runtimeContext.skills) { + await cp(skill.bundle.rootPath, join(scopedRunnerdStateRoot(execution), "codex-home/skills", skill.runtimeName), { recursive: true }); + } } const input = { companyId, selectedSessionId: selection!.id, scopeDigest, source, storage, signal: AbortSignal.timeout(60_000) }; return { input, runId, issueId, agentId, leaseId, execution, witness, storage, mount, sourceRoot: scopedRunnerdStateRoot(execution) }; @@ -231,6 +242,105 @@ describeEmbeddedPostgres("owner-verified native harness filesystem independence" return { ...f, input: { ...f.input, mode: "web_evidence" as const } }; } + it("preserves all 927 local WEB skill files and validates their archive on a cold replacement", async () => { + const f = await controllerFixture(true, "many-web-files"); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const metadata = { ...lease!.metadata }; delete metadata.remoteExecutionTermination; + await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.leaseId)); + const originalLeases = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId)); + const result = await preserveNativeHarnessFilesystemIndependence(db, { ...f.input, mode: "web_evidence" }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const receipt = run!.runnerProfileJson!.nativeHarnessFilesystemIndependence as { + web: { entries: Array<{ path: string; type: string }> }; controller: { archive: { id: string } }; + }; + expect(receipt.web.entries.length).toBeGreaterThan(927); + const [archive] = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, receipt.controller.archive.id)); + const target = join(f.mount, "cold-many-web-files"); + expect(await restoreDurableNativeHarnessBackup({ db, storage: f.storage, companyId: f.input.companyId, + scopeDigest: archive!.scopeDigest, backupRoot: target })).toBe(true); + if (!("runtimeContext" in f.execution)) throw new Error("fixture runtime context missing"); + const files: Array<{ relative: string; sha256: string }> = []; + for (const skill of f.execution.runtimeContext.skills) { + const manifest = JSON.parse(await readFile(join(f.mount, "instances/default/runtime-context-assets/manifests", `${skill.bundle.digest}.json`), "utf8")); + for (const file of manifest.files as Array<{ path: string; sha256: string }>) { + files.push({ relative: join("codex-home/skills", skill.runtimeName, file.path), sha256: file.sha256 }); + } + } + expect(files).toHaveLength(927); + for (let offset = 0; offset < files.length; offset += 16) { + await Promise.all(files.slice(offset, offset + 16).map(async file => { + const restored = join(target, "current/web-home", file.relative); + const [originalBytes, restoredBytes, stat] = await Promise.all([ + readFile(join(f.sourceRoot, file.relative)), readFile(restored), lstat(restored), + ]); + expect(restoredBytes).toEqual(originalBytes); + expect(createHash("sha256").update(originalBytes).digest("hex")).toBe(file.sha256); + expect(stat.mode & 0o777).toBe(0o444); + })); + } + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.input.companyId))).toEqual(originalLeases); + vi.stubEnv("PAPERCLIP_HOME", join(f.mount, "absent-many-web-files")); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(f.mount, "absent-many-web-files/state")); vi.stubEnv("RAILWAY_VOLUME_ID", ""); + vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("database validation must not read storage")); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(result.receiptDigest).toMatch(/^[a-f0-9]{64}$/); + }, 120_000); + + function webDescriptor(entries: NativeControllerHistoryManifestEntry[]) { + return nativeControllerEvidenceDescriptor({ companyId: "company", issueId: "issue", runId: "run", controllerDigest: null, + supervisorDigest: null, purpose: "filesystem_independence", web: { bindingDigest: "a".repeat(64), entries } }); + } + + it("keeps the existing 128-entry WEB manifest bytes unchanged", () => { + const entries: NativeControllerHistoryManifestEntry[] = [{ path: "web-home", mode: 0o700, type: "directory" }, + ...Array.from({ length: 127 }, (_, index) => ({ path: `web-home/entry-${index}`, mode: 0o700, type: "directory" as const }))]; + const descriptor = webDescriptor(entries); + const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: "company", issueId: "issue", + runId: "run", reason: "original_volume_source_evidence", files: {}, web: { entries, bindingDigest: "a".repeat(64), controllerAbsent: true } }); + expect(descriptor.manifest).toBe(manifest); + expect(descriptor.manifestSha256).toBe(`sha256:${createHash("sha256").update(manifest).digest("hex")}`); + }); + + it.each(["entry-count", "metadata-bytes", "manifest-bytes", "path-bytes", "file-bytes", "total-bytes"])("refuses a WEB manifest beyond its %s bound", control => { + const entries: NativeControllerHistoryManifestEntry[] = [{ path: "web-home", mode: 0o700, type: "directory" }]; + if (control === "entry-count") entries.push(...Array.from({ length: NATIVE_CONTROLLER_WEB_LIMITS.entries }, (_, index) => ({ + path: `web-home/entry-${index}`, mode: 0o700, type: "directory" as const, + }))); + if (control === "metadata-bytes") entries.push(...Array.from({ length: 1_050 }, (_, index) => ({ + path: `web-home/${index}-${"x".repeat(3_990)}`, mode: 0o700, type: "directory" as const, + }))); + if (control === "manifest-bytes") { + entries.push(...Array.from({ length: NATIVE_CONTROLLER_WEB_LIMITS.entries - 1 }, (_, index) => ({ + path: `web-home/${index}-${"x".repeat(240)}`, mode: 0o444, type: "file" as const, size: 0, sha256: "b".repeat(64), + }))); + expect(Buffer.byteLength(JSON.stringify(entries))).toBeLessThan(NATIVE_CONTROLLER_WEB_LIMITS.metadataBytes); + } + if (control === "path-bytes") entries.push({ path: `web-home/${"x".repeat(NATIVE_CONTROLLER_WEB_LIMITS.pathBytes)}`, mode: 0o700, type: "directory" }); + if (control === "file-bytes") entries.push({ path: "web-home/oversized", mode: 0o600, type: "file", size: 16 * 1024 * 1024 + 1, sha256: "b".repeat(64) }); + if (control === "total-bytes") entries.push(...Array.from({ length: 5 }, (_, index) => ({ path: `web-home/file-${index}`, + mode: 0o600, type: "file" as const, size: 16 * 1024 * 1024, sha256: "b".repeat(64) }))); + expect(() => webDescriptor(entries)).toThrow("native_controller_archive_authority_invalid"); + }); + + it.each(["entry-count", "file-bytes"])("retains oversized WEB filesystem evidence without publication for %s", async control => { + const f = await webFixture(), directory = join(f.sourceRoot, "codex-home/oversized"); + await mkdir(directory, { mode: 0o700 }); + if (control === "entry-count") { + for (let offset = 0; offset <= NATIVE_CONTROLLER_WEB_LIMITS.entries; offset += 32) { + await Promise.all(Array.from({ length: Math.min(32, NATIVE_CONTROLLER_WEB_LIMITS.entries + 1 - offset) }, (_, index) => + writeFile(join(directory, `entry-${offset + index}`), "", { mode: 0o600 }))); + } + } else { + const file = join(directory, "oversized"); await writeFile(file, "", { mode: 0o600 }); + await truncate(file, 16 * 1024 * 1024 + 1); + } + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("unverified"); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run!.runnerProfileJson!.nativeHarnessFilesystemIndependence).toBeUndefined(); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.runId, f.runId))).toEqual([]); + expect(await readFile(f.controllerPath, "utf8")).toBe(f.raw); + }); + it("preserves a genuinely absent controller with its supervisor and 267 durable checkpoint parts", async () => { const f = await webFixture(267); await rm(f.controllerPath); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.ts index f8487b79922..452f5bce780 100644 --- a/server/src/services/native-runtime/native-harness-filesystem-independence.ts +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.ts @@ -28,7 +28,7 @@ import { assertOriginalHarnessVolumeSource, type OriginalHarnessVolumeCapability import { isKnownNativeSessionRootEntry } from "./native-session-root-entries.js"; import { readLegacyNativeControllerArchiveSource, readLegacyNativeControllerArchiveDigests } from "./native-controller-migration.js"; import { archiveNativeControllerSourceEvidence, legacyNativeControllerArchive, nativeControllerEvidenceDescriptor, - nativeControllerHistoryManifest, nativeControllerWebManifest, type NativeControllerWebEntry, + nativeControllerHistoryManifest, nativeControllerWebManifest, NATIVE_CONTROLLER_WEB_LIMITS, type NativeControllerWebEntry, type NativeControllerHistoryEntry, type NativeControllerHistoryManifestEntry } from "./native-controller-archive.js"; import { nativeControllerArchiveCoordinatorDigest, nativeControllerArchiveProfileDigest, type SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; import { loadNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; @@ -479,12 +479,14 @@ function webProfile(facts: Awaited>) { function validWebBinding(facts: Awaited>, web: WebBinding): boolean { try { if (!web || typeof web.rootPresent !== "boolean" || typeof web.controllerAbsent !== "boolean" || !Array.isArray(web.entries) - || !Array.isArray(web.skills) || !Array.isArray(web.excluded) || web.entries.length > 128 || web.excluded.length > 128 + || !Array.isArray(web.skills) || !Array.isArray(web.excluded) + || web.entries.length + web.excluded.length > NATIVE_CONTROLLER_WEB_LIMITS.entries || web.profileDigest !== digest(webProfile(facts)) || digest(web.skills) !== digest(webSkills(facts))) return false; const profiles = webProfile(facts), manifests = new Map(web.skills.map(skill => [skill.bundleDigest, skill.manifestDigest])); for (const [bundle, manifest] of manifests) if (!web.entries.some(entry => entry.path === `web-manifests/${bundle}.json` && entry.type === "file" && entry.sha256 === manifest)) return false; if (web.entries.some(entry => { + if (Buffer.byteLength(entry.path) > NATIVE_CONTROLLER_WEB_LIMITS.pathBytes) return true; const [prefix, name, ...rest] = entry.path.split("/"); if (prefix === "web-supervisor.json") return name !== undefined || entry.type !== "file"; if (prefix === "web-manifests") return name === undefined ? entry.type !== "directory" @@ -499,10 +501,11 @@ function validWebBinding(facts: Awaited>, web: return web.excluded.every(entry => { const [name, ...rest] = entry.path.split("/"), rel = rest.join("/"), profile = profiles.find(profile => profile.name === name); assertSkillSnapshotPath(entry.path); - return profile?.excludeEntries.some(excluded => rel === excluded || rel.startsWith(`${excluded}/`)) + return Buffer.byteLength(entry.path) <= NATIVE_CONTROLLER_WEB_LIMITS.pathBytes + && profile?.excludeEntries.some(excluded => rel === excluded || rel.startsWith(`${excluded}/`)) && ["directory", "file"].includes(entry.type) && Number.isInteger(entry.mode) && (entry.mode & ~0o777) === 0 && Number.isSafeInteger(entry.size) && entry.size >= 0; - }); + }) && Buffer.byteLength(JSON.stringify({ entries: web.entries, excluded: web.excluded })) <= NATIVE_CONTROLLER_WEB_LIMITS.metadataBytes; } catch { return false; } } @@ -513,7 +516,7 @@ async function captureWebEvidence(facts: Awaited { const names: string[] = []; for await (const entry of await opendir(path)) { - signal.throwIfAborted(); if (names.length >= 128) fail(); names.push(entry.name); + signal.throwIfAborted(); if (names.length >= NATIVE_CONTROLLER_WEB_LIMITS.entries) fail(); names.push(entry.name); } return names.sort(); }; @@ -540,7 +543,8 @@ async function captureWebEvidence(facts: Awaited => { - signal.throwIfAborted(); if (++count > 128) fail(); assertSkillSnapshotPath(path); + signal.throwIfAborted(); if (++count > NATIVE_CONTROLLER_WEB_LIMITS.entries + || Buffer.byteLength(path) > NATIVE_CONTROLLER_WEB_LIMITS.pathBytes) fail(); assertSkillSnapshotPath(path); const s = await lstat(file); if (s.isSymbolicLink() || !s.isDirectory() && !s.isFile() || s.dev !== instance.dev || s.uid !== instance.uid || (s.mode & 0o7000) !== 0) fail(); metadata.push([path, s.dev, s.ino, s.size, s.mode, s.mtimeMs, s.ctimeMs]); @@ -590,7 +594,7 @@ async function captureWebEvidence(facts: Awaited 128) fail(); + if (entries.length + excluded.length > NATIVE_CONTROLLER_WEB_LIMITS.entries) fail(); const material = await readLegacyNativeControllerArchiveDigests(facts.source); const binding: WebBinding = { entries: nativeControllerWebManifest(entries), skills, rootPresent: rootStat !== null, profileDigest: digest(profiles), controllerAbsent: material.controllerDigest === undefined, excluded };