diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 1e11ff657e7..5bcf737602d 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -49,6 +49,38 @@ failed results and recovery blockers stay unchanged. A successful retained session without artifact proof still blocks migration. The migration receipt reports failed-session archives separately from imported recovery authorities. +An internal maintenance owner can separately record +`runnerProfileJson.nativeHarnessFilesystemIndependence` when an original-volume +inspection finds no uncovered provider or Runner payload. This is not a backup +or a successful recovery. It requires a fresh, independently reviewed provider +attachment witness and a private nonce anchor on that original volume. The +anchor survives normal releases; current deployment and mount identities are +verified afresh. Replacement hosts cannot create the receipt from missing files. + +Publication binds the retained session, canonical newest resume source, frozen +inputs and checkpoints, every associated lease and destroyed-provider receipt, +released controller/finalizer ownership, and existing immutable skill references. +Any surviving skill files must match their admitted package bytes and modes. +Surviving controller and supervisor files require separate authenticated raw-byte +archives through the existing encrypted backup ledger. This evidence-only path +also accepts a settled successful source whose controller records match its +durable checkpoint. Its `original_volume_source_evidence` manifest reason does +not grant provider recovery or Runner execution authority. Publication checks +the source bytes again and retains the first authenticated archive during a +concurrent capture. The receipt binds the exact archive row and checkpoint parts. +Source locks and a final comparison fence publication; original files, stamps, +warnings, failed statuses and session pointers remain unchanged. Startup can +validate the published receipt using only database bindings, without the old +volume, a fresh source witness, or storage reads. Changed or missing bindings +invalidate it. Existing resume and replacement policies remain authoritative, +including governed-wait failures and warm-session refusals. No provider history +is reconstructed or declared recovered, and uncertain provider sources remain +enrollment blockers. + +Receipt validation adds bounded per-scope database reads to startup eligibility. +It does not add a recurring scan or network work. Qualification must measure the +selected cohort's startup latency on the serving revision before enrollment. + ### Proactive agent chat messages Migration `0419_slippery_impossible_man.sql` adds an agent-scoped unique constraint diff --git a/server/src/services/native-runtime/native-controller-archive-source.ts b/server/src/services/native-runtime/native-controller-archive-source.ts index 5101dbe71b5..7f55ed1acf1 100644 --- a/server/src/services/native-runtime/native-controller-archive-source.ts +++ b/server/src/services/native-runtime/native-controller-archive-source.ts @@ -7,12 +7,14 @@ import { readLegacyNativeControllerArchiveDigests, readLegacyNativeControllerArc import { tryLockNativeWorkspaceFinalization } from "./native-workspace-finalization-ownership.js"; import { archiveFailedLegacyNativeController } from "./native-controller-archive.js"; import type { StorageService } from "../../storage/types.js"; +import { assertOriginalHarnessVolumeSource, type OriginalHarnessVolumeCapability } from "./native-harness-original-volume.js"; export type SettledControllerArchiveOwner = { kind: "settled_controller_archive"; companyId: string; issueId: string; agentId: string; runId: string; sessionId: string; selectedSessionId: string; selectedLastRunId: string; profileDigest: string; coordinatorDigest: string; controllerDigest: string; supervisorDigest: string | null; + filesystemIndependence?: { source: OriginalHarnessVolumeCapability; scopeDigest: string; factsDigest: string }; }; const fileDigest = (value: string) => createHash("sha256").update(value).digest("hex"); @@ -31,6 +33,10 @@ export function nativeControllerArchiveCoordinatorDigest(value: { export async function assertSettledControllerArchiveSource(db: Db, owner: SettledControllerArchiveOwner) { function fail(): never { throw new Error("native_controller_archive_source_unsettled"); } + if (owner.filesystemIndependence) { + await assertOriginalHarnessVolumeSource(owner.filesystemIndependence.source); + await (await import("./native-harness-filesystem-independence.js")).assertNativeHarnessFilesystemArchiveSource(db, owner); + } await db.execute(sql`set local lock_timeout = '1s'`); if (!await tryLockNativeWorkspaceFinalization(db, owner)) fail(); const [task] = await db.select().from(issues).where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId))).for("update"); @@ -50,7 +56,9 @@ export async function assertSettledControllerArchiveSource(db: Db, owner: Settle const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.agentId, owner.agentId), eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.nativeSessionId, owner.sessionId), eq(heartbeatRuns.runtimeMode, "native"))).for("share"); - if (!selection || !selectedRun || !run?.finishedAt || run.status !== "failed" + if (!selection || !selectedRun || !run?.finishedAt + || (owner.filesystemIndependence ? !["failed", "succeeded"].includes(run.status) + || run.status === "succeeded" && (coordinator.phase !== "committed" || !coordinator.resultId) : run.status !== "failed") || nativeControllerArchiveProfileDigest(run.runnerProfileJson) !== owner.profileDigest) fail(); const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, owner.companyId), inArray(heartbeatRuns.status, ["queued", "running"]), or(eq(heartbeatRuns.nativeIssueId, owner.issueId), diff --git a/server/src/services/native-runtime/native-controller-archive.ts b/server/src/services/native-runtime/native-controller-archive.ts index d47eda47aba..3a2402ff8ae 100644 --- a/server/src/services/native-runtime/native-controller-archive.ts +++ b/server/src/services/native-runtime/native-controller-archive.ts @@ -8,12 +8,14 @@ import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import type { SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; -type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string }; +type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string; + purpose?: "filesystem_independence" }; const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); export function legacyNativeControllerArchive(source: Source) { const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId, - issueId: source.issueId, runId: source.runId, reason: "original_runner_artifact_unproven", + issueId: source.issueId, runId: source.runId, reason: source.purpose === "filesystem_independence" + ? "original_volume_source_evidence" : "original_runner_artifact_unproven", files: { "control-plane-state.json": sha256(source.controller), ...(source.supervisor === undefined ? {} : { "supervisor.json": sha256(source.supervisor) }) } }); return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` }; @@ -26,6 +28,24 @@ export async function hasArchivedLegacyNativeController(db: Db, input: Source & export async function archiveFailedLegacyNativeController(db: Db, input: Source & { storage: StorageService; writersStopped?: true; onlineOwner?: SettledControllerArchiveOwner; signal: AbortSignal; +}) { + if (input.purpose || input.onlineOwner?.filesystemIndependence) throw new Error("native_controller_archive_authority_invalid"); + return persistLegacyControllerEvidence(db, input); +} + +export async function archiveNativeControllerSourceEvidence(db: Db, input: Source & { + storage: StorageService; onlineOwner: SettledControllerArchiveOwner; signal: AbortSignal; +}) { + if (input.purpose !== "filesystem_independence" || !input.onlineOwner.filesystemIndependence) { + throw new Error("native_controller_archive_authority_invalid"); + } + await db.transaction(async tx => (await import("./native-controller-archive-source.js")) + .assertSettledControllerArchiveSource(tx as unknown as Db, input.onlineOwner)); + return persistLegacyControllerEvidence(db, input); +} + +async function persistLegacyControllerEvidence(db: Db, input: Source & { + storage: StorageService; writersStopped?: true; onlineOwner?: SettledControllerArchiveOwner; signal: AbortSignal; }) { if (input.onlineOwner ? input.writersStopped || input.onlineOwner.kind !== "settled_controller_archive" || input.onlineOwner.companyId !== input.companyId || input.onlineOwner.issueId !== input.issueId diff --git a/server/src/services/native-runtime/native-harness-backup-migration.ts b/server/src/services/native-runtime/native-harness-backup-migration.ts index eb4260c3766..194afba4f64 100644 --- a/server/src/services/native-runtime/native-harness-backup-migration.ts +++ b/server/src/services/native-runtime/native-harness-backup-migration.ts @@ -10,9 +10,14 @@ import { MAX_NATIVE_HARNESS_BYTES, MAX_NATIVE_HARNESS_OBJECT_BYTES, nativeHarnes import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; import { NativeHarnessBackupSourceError, readNativeHarnessBackupSource } from "./native-harness-backup-source.js"; import { settledNativeHarnessBackupOwner } from "./native-harness-preservation-source.js"; +import { hasValidNativeHarnessFilesystemIndependence } from "./native-harness-filesystem-independence.js"; export async function hasLocalNativeHarnessObligations(db: Db): Promise { - const rows = await db.execute(sql`select 1 from agent_task_sessions s + let afterSession = "", afterScope = ""; + for (;;) { + const rows = await db.execute<{ company_id: string; selected_session_id: string; scope_digest: string; has_receipt: boolean }>(sql`select distinct + s.company_id, s.id::text as selected_session_id, coalesce(e.metadata->'nativeHarnessBackup'->>'sessionScopeSha256', '') as scope_digest, + r.runner_profile_json ? 'nativeHarnessFilesystemIndependence' as has_receipt from agent_task_sessions s join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id join environment_leases e on e.company_id = r.company_id and e.issue_id = r.native_issue_id and e.metadata->'nativeHarnessBackup'->>'normalizedSessionId' = r.native_session_id::text @@ -20,8 +25,14 @@ export async function hasLocalNativeHarnessObligations(db: Db): Promise and e.metadata->'nativeHarnessBackup'->>'schema' = 'paperclip.native-harness-backup-stamp.v2' and not exists (select 1 from native_harness_backups b where b.company_id = r.company_id and b.scope_digest = e.metadata->'nativeHarnessBackup'->>'sessionScopeSha256' and b.status = 'current') - limit 1`); - return rows.length > 0; + and (s.id::text, coalesce(e.metadata->'nativeHarnessBackup'->>'sessionScopeSha256', '')) > (${afterSession}, ${afterScope}) + order by selected_session_id, scope_digest limit 100`); + for (const row of rows) if (!row.has_receipt || !await hasValidNativeHarnessFilesystemIndependence(db, { + companyId: row.company_id, selectedSessionId: row.selected_session_id, scopeDigest: row.scope_digest, + })) return true; + if (rows.length < 100) return false; + afterSession = rows.at(-1)!.selected_session_id; afterScope = rows.at(-1)!.scope_digest; + } } const MAX_PARALLEL_BACKUPS = 4; diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index 6f0612efc18..d285e211b07 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -256,6 +256,16 @@ export async function persistNativeHarnessBackup(input: { sql`${heartbeatRuns.createdAt} >= (select created_at from heartbeat_runs where id = ${input.owner.runId}::uuid)`)).limit(1); if (successor) fail("owner_superseded"); await fence(tx, input.owner, inactiveSource, input.signal); + if ("kind" in input.owner && input.owner.kind === "settled_controller_archive" && input.owner.filesystemIndependence) { + const [existing] = await tx.select({ id: nativeHarnessBackups.id }).from(nativeHarnessBackups).where(and(scope(ref), + eq(nativeHarnessBackups.runId, input.owner.runId), eq(nativeHarnessBackups.manifestSha256, manifestSha256), + eq(nativeHarnessBackups.status, "current"))).limit(1); + if (existing) { + await tx.update(nativeHarnessBackups).set({ status: "superseded", updatedAt: new Date() }) + .where(and(eq(nativeHarnessBackups.id, ref.id), eq(nativeHarnessBackups.status, "pending"))); + return; + } + } await tx.update(nativeHarnessBackups).set({ status: "superseded", updatedAt: new Date() }) .where(and(scope(ref), eq(nativeHarnessBackups.status, "current"))); await tx.update(nativeHarnessBackups).set({ status: "current", updatedAt: new Date() }) @@ -266,6 +276,12 @@ export async function persistNativeHarnessBackup(input: { agentId: input.owner.agentId, runId: input.owner.runId, details: { scopeDigest: ref.scopeDigest, manifestSha256, backupId: ref.id, originalFilesRetained: true } }, publications); } + if ("kind" in input.owner && input.owner.kind === "settled_controller_archive" && input.owner.filesystemIndependence) { + await logActivity(tx as unknown as Db, { companyId: input.owner.companyId, actorType: "system", actorId: "native-harness-preservation", + action: "native_controller_source_evidence_preserved", entityType: "heartbeat_run", entityId: input.owner.runId, + agentId: input.owner.agentId, runId: input.owner.runId, details: { scopeDigest: ref.scopeDigest, manifestSha256, + backupId: ref.id, sourceCapabilityFingerprint: input.owner.filesystemIndependence.source.fingerprint } }, publications); + } }); for (const publication of publications) publishActivity(publication); } finally { key.fill(0); await removeNativeHarnessBackup(scratch); } @@ -324,10 +340,11 @@ export async function restoreDurableNativeHarnessBackup(input: { } export async function verifyDurableNativeHarnessBackup(input: { - db: Db; storage: StorageService; companyId: string; scopeDigest: string; manifestSha256: string; signal?: AbortSignal; + db: Db; storage: StorageService; companyId: string; scopeDigest: string; manifestSha256: string; backupId?: string; signal?: AbortSignal; }): Promise { const [row] = await input.db.select().from(nativeHarnessBackups).where(and(scope(input), - eq(nativeHarnessBackups.manifestSha256, input.manifestSha256), inArray(nativeHarnessBackups.status, ["current", "superseded"]))).limit(1); + eq(nativeHarnessBackups.manifestSha256, input.manifestSha256), input.backupId ? eq(nativeHarnessBackups.id, input.backupId) : undefined, + inArray(nativeHarnessBackups.status, ["current", "superseded"]))).limit(1); if (!row) return false; await receive(input.storage, reference(row), undefined, input.signal); return true; diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence-continuation.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence-continuation.integration.test.ts new file mode 100644 index 00000000000..3d3ae2327ef --- /dev/null +++ b/server/src/services/native-runtime/native-harness-filesystem-independence-continuation.integration.test.ts @@ -0,0 +1,145 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { afterEach, expect, it, vi } from "vitest"; +import { completionContracts, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { DurablePrpControlPlane, parseNativeExecutionInput, QUALIFIED_ACPX_PROFILES, QUALIFIED_ACPX_VERSION, type NativeSessionBackend, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; +import { createNativeControllerRecovery } from "./native-controller-recovery.js"; +import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { createRunnerdBackend, executePaperclipNativeSession, nativeControllerMigrationDescriptor, scopedRunnerdStateRoot } from "./native-session-executor.js"; +import { nativeToolContractFingerprintForTarget, rebindNativeSessionCheckpoint } from "./native-session-resume.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { nativeSha256 } from "./canonical.js"; + +const remoteBoundary = vi.hoisted(() => ({ + backend: vi.fn((_execution: unknown, _options: { codexTransportFactory?: (context: unknown) => unknown }) => ({})), + transport: vi.fn((_options: Record) => ({ transport: {} })), + packPath: "", binaryPath: "", +})); +vi.mock("../../vendor/paperclip-runner/index.js", async importOriginal => ({ + ...await importOriginal(), + createNativeSessionBackend: remoteBoundary.backend, + createRunnerdCodexTransport: remoteBoundary.transport, + bundledRemoteProviderPackManifestPath: () => remoteBoundary.packPath, + bundledRemoteRunnerBinary: () => remoteBoundary.binaryPath, +})); +vi.mock("./paperclip-runner-tool-authority.js", () => ({ PaperclipRunnerToolAuthority: class { + readonly binding = {}; + async definitions() { return []; } + close() {} +} })); + +describeEmbeddedPostgres("legacy harness continuation without provider-home files", () => { + const environment = nativeRestartFixture(); + const homes: string[] = []; + afterEach(async () => { vi.unstubAllEnvs(); for (const home of homes.splice(0)) await rm(home, { recursive: true, force: true }); }); + + it.each([["codex", "per_turn"], ["opencode", "per_turn"], ["codex", "warm"]] as const)("preserves explicit %s %s recovery policy through server and Runner on both homes", async (provider, lifecycle) => { + for (const policy of ["allow_replacement_after_resume_failure", "allow_replacement_after_governed_wait"] as const) { + const observations = []; + for (const pose of ["original", "empty"] as const) { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); + const f = await environment.seedRun(randomUUID()); + const { companyId, agentId, runtimeRoot } = environment.current; + const home = await mkdtemp(join(tmpdir(), "native-harness-independence-")); homes.push(home); + vi.stubEnv("PAPERCLIP_HOME", home); + const workspaceId = randomUUID(); + const [sourceRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [contract] = await f.db.select().from(completionContracts).where(eq(completionContracts.id, sourceRun!.completionContractId!)); + const execution = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, + provider: provider === "codex" ? { kind: "codex", model: null, approvalPolicy: "never" } : { kind: "opencode", model: "fixture/model", permissionMode: "allow" }, + binding: { companyId, agentId, issueId: f.issueId, runId: f.runId, executionWorkspaceId: workspaceId }, + task: { identifier: "LHI-1", title: "Retained provider work", description: null, prompt: "Continue the saved work.", workMode: "standard" }, + workspace: { cwd: runtimeRoot, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: f.native.normalizedSessionId, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server", protocolVersion: 1, lifecyclePolicy: lifecycle === "per_turn" ? { mode: "per_turn", idleTimeoutMs: null } : { mode: "warm", idleTimeoutMs: 60_000 } }, + completionContract: { id: contract!.id, sha256: contract!.canonicalSha256, schemaVersion: contract!.schemaVersion, contract: contract!.contractJson }, + runtimeContext: nativeRuntimeContextFixture(), credentialBindings: [], interactionResponses: [], + }); + const semanticResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: policy === "allow_replacement_after_governed_wait" ? "yielded" : "done", summary: "Prior provider result", completionClaim: { contractRevision: "1", objectiveSatisfied: policy !== "allow_replacement_after_governed_wait", criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [], ...(policy === "allow_replacement_after_governed_wait" ? { continuation: { kind: "response_wake", summary: "Continue after the answer", idempotencyKey: "fixture-answer" } } : {}) }; + const snapshot: PersistedNativeSession = { backendKind: "runner", driverKind: execution.session.driverKind, sessionId: "retained-driver-session", providerSessionId: "retained-provider-session", identity: { companyId, agentId, issueId: f.issueId, runId: f.runId, sessionId: f.native.normalizedSessionId }, cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], terminalTurns: [], lineage: [], semanticResult: semanticResult as never, goal: null }; + const fingerprint = nativeToolContractFingerprintForTarget("remote", provider); + const descriptor = nativeControllerMigrationDescriptor(execution, fingerprint); + const bootId = randomUUID(), leaseOwner = randomUUID(); + await f.db.update(nativeRunFinalizations).set({ controllerBootId: bootId, controllerGeneration: 1, controllerPid: 2_147_483_000, controllerProcessStartedAt: new Date(Date.now() - 60_000), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const core = new DurablePrpControlPlane({ stateDirectory: join(runtimeRoot, randomUUID()), identity: { runId: f.runId, normalizedSessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId, environmentLeaseId: f.native.environmentLeaseId, turnId: f.native.turnId, itemId: f.native.itemId }, expectedRunnerVersion: "independence-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + for (const type of ["run.prepare", "session.open", "turn.start", "runner.suspend"]) Object.assign(core.queueCommand(type), { status: "completed", result: { status: "completed" } }); + core.store.state.committedEvents.push({ sourceSeq: 1, sourceEventId: "fixture-terminal", eventType: "run.terminal", priority: 0, envelope: { ...core.store.state.identity }, deliveryCount: 1, logicalEffectCount: 1 }); + core.store.state.ackedSourceSeq = 1; + core.setCheckpointPersistence(nativeRunnerCheckpointWriter(f.db, { companyId, issueId: f.issueId, runId: f.runId, authorityRunId: f.runId, bootId, generation: 1, leaseOwner })); + await core.flushCheckpoint(); await core.stop(); + const [result] = await f.db.insert(nativeRunResults).values({ companyId, issueId: f.issueId, runId: f.runId, completionContractId: (await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.completionContractId!, serverFingerprint: randomUUID(), schemaStatus: "accepted", resultJson: semanticResult, canonicalSha256: "a".repeat(64) }).returning(); + const sourceProfile = { ...f.preparedProfile, nativeExecutionInput: execution, nativeToolContractFingerprint: fingerprint, sessionCheckpoint: snapshot, nativeControllerRecovery: createNativeControllerRecovery(snapshot, { configDigest: descriptor.configDigest, processConfigDigest: null }, { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId }) }; + await f.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: sourceProfile }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result!.id, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (pose === "original") { await mkdir(join(scopedRunnerdStateRoot(execution), "codex-home", "skills"), { recursive: true, mode: 0o700 }); await writeFile(join(home, "retained-unrelated-file"), "preserve"); } + const runId = randomUUID(); + const [run] = await f.db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: f.issueId, issueId: f.issueId, status: "running", runtimeMode: "native", nativeSessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId, completionContractId: sourceRun!.completionContractId, completionContractSha256: sourceRun!.completionContractSha256 }).returning(); + const native = await prepareNativeHeartbeatRun({ db: f.db, run: run!, issue: { id: f.issueId, title: "Retained provider work", description: null }, environmentLeaseId: randomUUID() }); + const current = { ...execution, binding: { ...execution.binding, runId } }; + const rebound = rebindNativeSessionCheckpoint({ previousRun: { id: f.runId, companyId, agentId, nativeSessionId: f.native.normalizedSessionId, runnerProfileJson: sourceProfile }, currentExecution: current, executionTargetKind: "remote" }); + expect(rebound?.providerRecoveryPolicy).toBe(policy); + const [prepared] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...prepared!.runnerProfileJson, nativeExecutionInput: current, sessionCheckpoint: rebound } }).where(eq(heartbeatRuns.id, runId)); + await f.db.insert(nativeRunFinalizations).values({ runId, companyId, issueId: f.issueId, phase: "observed" }); + await f.db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + const recover = vi.fn(async (saved: PersistedNativeSession) => ({ recovered: false as const, reason: "fixture provider source is absent" })); + const fresh = vi.fn(async () => { throw new Error("fixture replacement observed before provider turn"); }); + const describe = vi.fn(async () => ({ kind: "runner", name: "controlled provider boundary", version: "fixture", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } })); + const backend: NativeSessionBackend = { descriptor: describe, recoverSession: recover, openSession: fresh, openReplacementSession: fresh }; + const failure = await executePaperclipNativeSession({ db: f.db, execution: current, runnerInstanceId: native.runnerInstanceId, useRunnerd: true, backend, runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: runtimeRoot, environmentId: randomUUID(), leaseId: randomUUID(), providerKey: "fixture", reusableLeaseConfigured: true, runner: { execute: async () => { throw new Error("unexpected remote process execution"); } }, sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "new-fixture-sandbox" } } as never }).catch(error => error); + if (lifecycle === "warm") { + expect(String(failure)).toContain("runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate"); + expect(describe).not.toHaveBeenCalled(); expect(recover).not.toHaveBeenCalled(); expect(fresh).not.toHaveBeenCalled(); + const [retained] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [unchanged] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(retained!.runnerProfileJson).toEqual(sourceProfile); + expect(unchanged!.runnerProfileJson?.sessionCheckpoint).toEqual(rebound); + observations.push({ refusal: String(failure), providerSessionId: rebound!.providerSessionId, + policy: rebound!.providerRecoveryPolicy, replacementAttempts: 0 }); + continue; + } + expect(recover, String(failure)).toHaveBeenCalledOnce(); + expect(recover.mock.calls[0]![0]).toMatchObject({ providerSessionId: snapshot.providerSessionId, providerRecoveryPolicy: policy, identity: { runId } }); + expect(fresh).toHaveBeenCalledTimes(policy === "allow_replacement_after_resume_failure" ? 1 : 0); + const digest = `sha256:${"a".repeat(64)}`; + const artifacts = Object.fromEntries(Object.entries({ grokLauncher: "dist/providers/grok/launcher.cjs", nodeCommand: "node_modules/node/bin/node", + productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs" }).map(([key, path]) => [key, { path, sha256: digest }])); + const payload = { compatibility: { opencode: 1, acpx: 1 }, contractDigest: digest, target: { platform: "linux", architecture: "x64" }, + runnerSourceRevision: "a".repeat(40), pins: { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion, + opencode: "1.18.34", acpx: QUALIFIED_ACPX_VERSION, grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion, + claudeAcp: QUALIFIED_ACPX_PROFILES.claude.agentServerVersion, codexAcp: QUALIFIED_ACPX_PROFILES.codex.agentServerVersion }, + acpxProfileDigests: Object.fromEntries(["grok", "claude", "codex"].map(agent => [agent, QUALIFIED_ACPX_PROFILES[agent as "grok" | "claude" | "codex"].commandDigest])), + distDigest: digest, artifacts, bridgeDigest: `sha256:${createHash("sha256").update(`${digest}\n${digest}\n${digest}`).digest("hex")}` }; + const pack = { schema: "paperclip-runner/remote-provider-pack/v1", payload, digest: `sha256:${nativeSha256(payload)}` }; + remoteBoundary.packPath = join(home, "provider-pack.json"); remoteBoundary.binaryPath = join(home, "fixture-runnerd"); + const elf = Buffer.alloc(64); elf.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); elf.writeUInt16LE(62, 18); + await writeFile(remoteBoundary.packPath, JSON.stringify(pack)); await writeFile(remoteBoundary.binaryPath, elf); + const remoteExecute = vi.fn(async (command: { args?: string[] }) => ({ exitCode: command.args?.[1]?.includes("base64") ? 1 : 0, + timedOut: false, stderr: "", stdout: command.args?.[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", + binaryContractVersion: 2, durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: pack, immutable: true }], + }) : command.args?.[0] === "--version" ? "codex-cli 0.160.0" : command.args?.[1] === "uname -s; uname -m" ? "Linux\nx86_64\n" : "" })); + await createRunnerdBackend({ db: f.db, execution: current, runnerInstanceId: native.runnerInstanceId, runnerIngressAuthorized: true, runnerRemoteBinaryPath: remoteBoundary.binaryPath, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: randomUUID(), leaseId: randomUUID(), + providerKey: "fixture", reusableLeaseConfigured: true, effectiveCapabilities: { runnerWebSocketIngress: true }, + runner: { execute: remoteExecute, syncIn: vi.fn(async () => undefined) }, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "new-fixture-sandbox" } } as never }); + remoteBoundary.backend.mock.calls.at(-1)![1].codexTransportFactory!({ + persistedSession: { driverSessionId: snapshot.sessionId, providerSessionId: snapshot.providerSessionId }, providerRecoveryPolicy: policy }); + const transport = remoteBoundary.transport.mock.calls.at(-1)![0]; + await expect((transport.prepareExternalRunnerState as () => Promise)()).resolves.toBeUndefined(); + expect(transport.providerRecoveryPolicy).toBe(policy); + expect(transport.resumeProviderSession).toMatchObject({ providerSessionId: snapshot.providerSessionId }); + observations.push({ providerSessionId: recover.mock.calls[0]![0].providerSessionId, policy: recover.mock.calls[0]![0].providerRecoveryPolicy, replacementAttempts: fresh.mock.calls.length }); + } + expect(observations[0]).toEqual(observations[1]); + } + }, 60_000); +}); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts new file mode 100644 index 00000000000..0d0793a0eec --- /dev/null +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.integration.test.ts @@ -0,0 +1,402 @@ +import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { and, eq } from "drizzle-orm"; +import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared"; +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { activityLog, agentTaskSessions, agents, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb, environmentLeases, + heartbeatRuns, issues, nativeHarnessBackups, nativeRunnerCheckpoints, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; +import { describeEmbeddedPostgres } from "../../__tests__/helpers/native-restart.js"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { createStorageService } from "../../storage/service.js"; +import { canonicalNativeRuntimeContextDigest, captureLegacyDurablePrpCheckpoint, DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { materializeAsset } from "./runtime-context.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { preserveRuntimeSkillAssets } from "./runtime-skill-preservation.js"; +import { nativeControllerMigrationDescriptor, scopedRunnerdStateRoot } from "./native-session-executor.js"; +import { hasLocalNativeHarnessObligations } from "./native-harness-backup-migration.js"; +import { preserveNativeHarnessFilesystemIndependence, hasValidNativeHarnessFilesystemIndependence } from "./native-harness-filesystem-independence.js"; +import { originalHarnessVolumeAnchorPath, verifyOriginalHarnessVolumeSource, type OriginalHarnessVolumeAnchor, + type ReviewedOriginalHarnessVolumeWitness } from "./native-harness-original-volume.js"; +import { removeRuntimeAssetTree } from "./runtime-asset-retention.js"; +import { skillPackageStore } from "../skill-package-store.js"; +import { repositorySkillProjection } from "../skill-repository-package.js"; +import { skillFileBytes } from "../skill-snapshot.js"; +import { tryLockNativeWorkspaceFinalization } from "./native-workspace-finalization-ownership.js"; +import { loadNativeRunnerCheckpoint, nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { createNativeControllerRecovery } from "./native-controller-recovery.js"; +import { ensureNativeCompletionContract } from "./completion-contracts.js"; +import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; + +vi.mock("../../build-commit.js", () => ({ readBuildCommit: () => "a".repeat(40) })); + +describeEmbeddedPostgres("owner-verified native harness filesystem independence", () => { + let database: Awaited>, db: ReturnType; + const roots: string[] = []; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("native-harness-filesystem-"); db = createDb(database.connectionString); }, 90_000); + afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await db.delete(agentTaskSessions); for (const root of roots.splice(0)) await removeRuntimeAssetTree(root); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function fixture(kind: "unversioned" | "repository" | "package" | "none" = "unversioned") { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); + const mount = await realpath(await mkdtemp(join(tmpdir(), "native-harness-original-"))); roots.push(mount); + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); + const projectId = randomUUID(), serviceId = randomUUID(), environmentId = randomUUID(), volumeId = randomUUID(), deploymentId = randomUUID(); + for (const [key, value] of Object.entries({ PAPERCLIP_HOME: mount, PAPERCLIP_INSTANCE_ID: "default", RAILWAY_PROJECT_ID: projectId, + RAILWAY_SERVICE_ID: serviceId, RAILWAY_ENVIRONMENT_ID: environmentId, RAILWAY_VOLUME_ID: volumeId, + RAILWAY_DEPLOYMENT_ID: deploymentId, RAILWAY_VOLUME_MOUNT_PATH: mount, + PAPERCLIP_RUNNER_STATE_DIR: join(mount, "instances/default/runtime/paperclip-runner/durable-sessions") })) vi.stubEnv(key, value); + await mkdir(process.env.PAPERCLIP_RUNNER_STATE_DIR!, { recursive: true, mode: 0o700 }); + await mkdir(join(mount, "instances/default/recovery"), { recursive: true, mode: 0o700 }); + const m = await lstat(mount), instance = await lstat(join(mount, "instances/default")); + const anchor: OriginalHarnessVolumeAnchor = { schema: "paperclip.native-harness-original-volume-anchor.v1", nonce: "b".repeat(64), + projectId, serviceId, environmentId, volumeId, volumeInstanceId: randomUUID(), mountPath: mount, + mountIdentity: { dev: m.dev, ino: m.ino }, instanceIdentity: { dev: instance.dev, ino: instance.ino }, + createdFrom: { deploymentId: randomUUID(), commit: "c".repeat(40), providerWitnessSha256: "d".repeat(64) }, createdAt: new Date().toISOString() }; + const bytes = Buffer.from(JSON.stringify(anchor)); await writeFile(originalHarnessVolumeAnchorPath(), bytes, { mode: 0o600 }); + const witness: ReviewedOriginalHarnessVolumeWitness = { projectId, serviceId, environmentId, volumeId, volumeInstanceId: anchor.volumeInstanceId, + mountPath: mount, currentDeploymentId: deploymentId, currentCommit: "a".repeat(40), currentMountIdentity: anchor.mountIdentity, + currentInstanceIdentity: anchor.instanceIdentity, observedAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 120_000).toISOString(), + anchorSha256: createHash("sha256").update(bytes).digest("hex"), providerWitnessSha256: "e".repeat(64) }; + const source = await verifyOriginalHarnessVolumeSource(witness); + const storage = createStorageService({ ...createLocalDiskStorageProvider(join(mount, "objects")), id: "s3" }); + await db.insert(companies).values({ id: companyId, name: "Original-volume preservation", issuePrefix: `F${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Retained source" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Retain prior failure", status: "blocked" }); + const runtimeContext = nativeRuntimeContextFixture(); + if (kind !== "none") { + let versionId: string | null = null; + const files: CompanySkillVersionFileInventoryEntry[] = [{ path: "SKILL.md", kind: "skill", content: "# original immutable skill\n" }, + { path: "tool.sh", kind: "other" as const, content: "#!/bin/sh\nexit 0\n", executable: true }]; + let projected = files; + if (kind === "repository") { + files[0]!.content = "---\nname: retained\ndescription: retained\n---\n# original immutable skill\n"; + projected = repositorySkillProjection(files, "SKILL.md"); + } + if (kind === "repository" || kind === "package") { + const packageId = await skillPackageStore(db, storage).prepare(companyId, files, true); + const [skill] = await db.insert(companySkills).values({ companyId, key: "retained", slug: "retained", name: "Retained", markdown: projected[0]!.content }).returning(); + const [version] = await db.insert(companySkillVersions).values({ companyId, companySkillId: skill!.id, revisionNumber: 1, + ...(kind === "repository" ? { repositorySnapshotId: packageId, repositorySkillPath: "SKILL.md" } : { packageSnapshotId: packageId }), fileInventory: [] }).returning(); + versionId = version!.id; + } + const bundle = await materializeAsset(projected.map(file => ({ path: file.path, content: skillFileBytes(file), mode: file.executable ? 0o555 : 0o444 }))); + runtimeContext.skills = [{ key: "retained", runtimeName: "retained", versionId, bundle }]; + runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext); + } + const execution = parseNativeExecutionInput({ schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, + binding: { companyId, agentId, issueId, runId, executionWorkspaceId: runId }, provider: { kind: "codex", model: null }, + task: { identifier: "FS-1", title: "Retain failure", description: null, prompt: "Retain failure", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: sessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: "contract", sha256: "saved", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Retain failure", criteria: [{ id: "retain", requirement: "Retain failure" }] } }, + runtimeContext, interactionResponses: [], credentialBindings: [] }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId: runnerId, + runtimeMode: "native", status: "failed", finishedAt: new Date(), processPid: 1234, processStartedAt: new Date(0), + runnerProfileJson: { nativeExecutionInput: execution, nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"), retainedWarning: "provider resume remains failed", sessionCheckpoint: { + backendKind: "runner", driverKind: "codex_app_server", sessionId: "saved-provider-driver", providerSessionId: "saved-provider-session", + identity: { companyId, agentId, issueId, runId, sessionId }, activeTurnId: null, cursor: "2", goal: null, + } } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "terminal_failure", recoveryState: "blocked" }); + const [selection] = await db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner", taskKey: issueId, + lastRunId: runId, lastError: "existing provider failure" }).returning(); + const scopeDigest = basename(scopedRunnerdStateRoot(execution)), leaseId = randomUUID(), providerLeaseId = `destroyed-${randomUUID()}`; + await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: runId, provider: "daytona", providerLeaseId, + status: "released", cleanupStatus: "success", releasedAt: new Date(), metadata: { + nativeHarnessBackup: { schema: "paperclip.native-harness-backup-stamp.v2", normalizedSessionId: sessionId, runnerInstanceId: runnerId, + sessionScopeSha256: scopeDigest, sourceProviderLeaseId: providerLeaseId, authorizedProviderLeaseId: providerLeaseId, + manifestSha256: `sha256:${"f".repeat(64)}`, completedAt: new Date().toISOString() }, + remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId, runId, leaseId, provider: "daytona", providerLeaseId, + state: "destroyed", confirmedAt: new Date().toISOString() }, + } }); + if (kind !== "none") { + if (kind === "unversioned") await preserveRuntimeSkillAssets(db, { companyId, runId }, storage); + await mkdir(join(scopedRunnerdStateRoot(execution), "codex-home/skills"), { recursive: true, mode: 0o700 }); + await cp(runtimeContext.skills[0]!.bundle.rootPath, join(scopedRunnerdStateRoot(execution), "codex-home/skills/retained"), { recursive: true }); + } + const input = { companyId, selectedSessionId: selection!.id, scopeDigest, source, storage, signal: AbortSignal.timeout(60_000) }; + return { input, runId, issueId, agentId, leaseId, execution, witness, storage, mount, sourceRoot: scopedRunnerdStateRoot(execution) }; + } + + async function controllerFixture() { + const f = await fixture(); + const completion = await ensureNativeCompletionContract({ db, companyId: f.input.companyId, + issue: { id: f.issueId, title: "Retain controller evidence", description: null }, actorId: f.agentId }); + const execution = parseNativeExecutionInput({ ...f.execution, session: { ...f.execution.session, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } }, + completionContract: { id: completion.row.id, sha256: completion.row.canonicalSha256, + schemaVersion: completion.row.schemaVersion, contract: completion.contract } }); + const [prior] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const snapshot = prior!.runnerProfileJson!.sessionCheckpoint; + const descriptor = nativeControllerMigrationDescriptor(execution, nativeToolContractFingerprintForTarget("remote", "codex")); + const bootId = randomUUID(), leaseOwner = randomUUID(); + await db.update(heartbeatRuns).set({ status: "running", finishedAt: null, completionContractId: completion.row.id, + completionContractSha256: completion.row.canonicalSha256 }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "observed", controllerBootId: bootId, controllerGeneration: 1, + leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const identity = { runId: f.runId, normalizedSessionId: execution.session.normalizedSessionId!, runnerInstanceId: prior!.runnerInstanceId!, + environmentLeaseId: f.leaseId, turnId: randomUUID(), itemId: randomUUID() }; + const control = new DurablePrpControlPlane({ stateDirectory: join(f.sourceRoot, "control-plane"), identity, + expectedRunnerVersion: "raw-evidence-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + for (const type of ["run.prepare", "session.open", "turn.start", "runner.suspend"]) { + Object.assign(control.queueCommand(type), { status: "completed", result: { status: "completed" } }); + } + control.store.state.committedEvents.push({ sourceSeq: 1, sourceEventId: "raw-stream", eventType: "item.delta", priority: 1, + envelope: { ...identity, text: "Raw stream bytes absent from control-v1 records" }, deliveryCount: 1, logicalEffectCount: 1 }, + { sourceSeq: 2, sourceEventId: "raw-terminal", eventType: "run.terminal", priority: 0, + envelope: { ...identity }, deliveryCount: 1, logicalEffectCount: 1 }); + control.store.state.ackedSourceSeq = 2; + const raw = JSON.stringify(control.store.state); + control.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, { companyId: f.input.companyId, issueId: f.issueId, + runId: f.runId, authorityRunId: f.runId, bootId, generation: 1, leaseOwner })); + await control.flushCheckpoint(); + await control.stop(); + const controllerPath = join(f.sourceRoot, "control-plane/control-plane-state.json"); + await writeFile(controllerPath, raw, { mode: 0o600 }); + const supervisor = JSON.stringify({ schema: "paperclip.native-session-supervisor.v1", configDigest: descriptor.configDigest, + processConfigDigest: null, snapshot }); + await mkdir(join(descriptor.supervisorPath, ".."), { recursive: true, mode: 0o700 }); + await writeFile(descriptor.supervisorPath, supervisor, { mode: 0o600 }); + const [result] = await db.insert(nativeRunResults).values({ companyId: f.input.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: completion.row.id, serverFingerprint: "raw-evidence-fixture", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "a".repeat(64) }).returning(); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: { ...prior!.runnerProfileJson, + nativeExecutionInput: execution, nativeControllerRecovery: createNativeControllerRecovery(snapshot as never, + { configDigest: descriptor.configDigest, processConfigDigest: null }, + { runId: f.runId, runnerInstanceId: prior!.runnerInstanceId!, normalizedSessionId: execution.session.normalizedSessionId! }) } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result!.id, recoveryState: null, + leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, f.runId)); + return { ...f, execution, controllerPath, raw, supervisor, supervisorPath: descriptor.supervisorPath }; + } + + it("publishes only independence and validates on an empty replacement without the source capability", async () => { + const f = await fixture(); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + expect(await hasLocalNativeHarnessObligations(db)).toBe(true); + const [unrelated] = await db.insert(heartbeatRuns).values({ companyId: f.input.companyId, agentId: f.agentId, status: "running", runtimeMode: "native" }).returning(); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasLocalNativeHarnessObligations(db)).toBe(false); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(after).toEqual({ ...before, runnerProfileJson: { ...before!.runnerProfileJson, + nativeHarnessFilesystemIndependence: after!.runnerProfileJson!.nativeHarnessFilesystemIndependence } }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]).toEqual(lease); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", recoveryState: "blocked", leaseOwner: null }); + expect(await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId))).toEqual([]); + expect(await readFile(join(f.sourceRoot, "codex-home/skills/retained/SKILL.md"), "utf8")).toBe("# original immutable skill\n"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, unrelated!.id)))[0]!.status).toBe("running"); + const replacement = await realpath(await mkdtemp(join(tmpdir(), "native-harness-replacement-"))); roots.push(replacement); + vi.stubEnv("PAPERCLIP_HOME", replacement); vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(replacement, "state")); + vi.stubEnv("RAILWAY_VOLUME_ID", ""); vi.stubEnv("RAILWAY_DEPLOYMENT_ID", randomUUID()); + const get = vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("startup must not read objects")); + expect(await hasLocalNativeHarnessObligations(db)).toBe(false); expect(get).not.toHaveBeenCalled(); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified"))))).toHaveLength(1); + }); + + it.each(["history", "changed-skill", "backup-present", "unknown-provider", "owned-finalizer", "physical-writer", "storage-failure", "changed-binding"])("refuses %s without publishing or changing recovery", async control => { + const f = await fixture(), where = eq(heartbeatRuns.id, f.runId); + if (control === "history") await writeFile(join(f.sourceRoot, "codex-home/history.jsonl"), "retained conversation"); + if (control === "changed-skill") { await chmod(join(f.sourceRoot, "codex-home/skills/retained"), 0o700); await writeFile(join(f.sourceRoot, "codex-home/skills/retained/extra"), "uncovered user bytes"); } + if (control === "backup-present") await mkdir(join(f.sourceRoot, "failover-backups/current"), { recursive: true }); + if (control === "unknown-provider") await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, f.leaseId)); + if (control === "owned-finalizer") await db.update(nativeRunFinalizations).set({ leaseOwner: "expired-but-owned", leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (control === "physical-writer") { const [r] = await db.select().from(heartbeatRuns).where(where); await db.update(heartbeatRuns).set({ runnerProfileJson: { ...r!.runnerProfileJson, nativeWorkspaceFinalizationOwner: { token: "still-owned" } } }).where(where); } + if (control === "storage-failure") vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("unavailable object")); + if (control === "changed-binding") { const get = f.storage.getObject.bind(f.storage); let changed = false; + vi.spyOn(f.storage, "getObject").mockImplementation(async (...args) => { const result = await get(...args); if (!changed) { changed = true; + const [row] = await db.select().from(heartbeatRuns).where(where); await db.update(heartbeatRuns).set({ runnerProfileJson: { ...row!.runnerProfileJson, retainedWarning: "newer warning" } }).where(where); } return result; }); } + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(where))[0]!.runnerProfileJson).not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect(await hasLocalNativeHarnessObligations(db)).toBe(true); + }); + + it.each(["checkpoint", "lease", "package", "retained-selection", "malformed-stamp"])("invalidates a published receipt after changed %s", async control => { + const f = await fixture(); await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + if (control === "checkpoint") { const [r] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...r!.runnerProfileJson, sessionCheckpoint: { changed: true } } }).where(eq(heartbeatRuns.id, f.runId)); } + if (control === "lease") await db.update(environmentLeases).set({ metadata: {} }).where(eq(environmentLeases.id, f.leaseId)); + if (control === "package") await db.update(companySkillRepositorySnapshots).set({ storage: null }).where(eq(companySkillRepositorySnapshots.companyId, f.input.companyId)); + if (control === "retained-selection") await db.update(agentTaskSessions).set({ lastRunId: null }).where(eq(agentTaskSessions.id, f.input.selectedSessionId)); + if (control === "malformed-stamp") { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const stamp = { ...(lease!.metadata!.nativeHarnessBackup as Record) }; delete stamp.sessionScopeSha256; + await db.update(environmentLeases).set({ metadata: { ...lease!.metadata, nativeHarnessBackup: stamp } }).where(eq(environmentLeases.id, f.leaseId)); + expect(await hasLocalNativeHarnessObligations(db)).toBe(true); + } + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + }); + it.each(["repository", "package"] as const)("uses the exact retained %s version rather than today's head", async kind => { + const f = await fixture(kind); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeRuntimeSkillPreservation"); + await db.delete(companySkillVersions).where(eq(companySkillVersions.companyId, f.input.companyId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + }); + it("does not publish across the actual physical-finalization lock", async () => { + const f = await fixture("none"); + let release!: () => void, acquired!: () => void; + const held = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { acquired = resolve; }); + const owner = db.transaction(async tx => { expect(await tryLockNativeWorkspaceFinalization(tx as unknown as typeof db, + { companyId: f.input.companyId, runId: f.runId })).toBe(true); acquired(); await held; }); + try { await ready; await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("unverified"); } + finally { release(); await owner; } + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + }); + it("retains the first publication through concurrent attempts and an ordinary retry", async () => { + const f = await fixture(); + const results = await Promise.allSettled([preserveNativeHarnessFilesystemIndependence(db, f.input), preserveNativeHarnessFilesystemIndependence(db, f.input)]); + expect(results.filter(result => result.status === "fulfilled").length).toBeGreaterThan(0); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(before); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + }); + it.each([false, true])("retains or refuses the existing receipt when an earlier capture finishes late (invalid: %s)", async invalid => { + const f = await fixture(); + let release!: () => void, entered!: () => void, delay = true; + const blocked = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { entered = resolve; }); + const get = f.storage.getObject.bind(f.storage); + vi.spyOn(f.storage, "getObject").mockImplementation(async (...args) => { + if (delay) { delay = false; entered(); await blocked; } + return get(...args); + }); + const late = preserveNativeHarnessFilesystemIndependence(db, f.input); + try { + await ready; + const winner = await preserveNativeHarnessFilesystemIndependence(db, f.input); + let [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + if (invalid) { + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...published!.runnerProfileJson, + nativeHarnessFilesystemIndependence: { ...(published!.runnerProfileJson!.nativeHarnessFilesystemIndependence as Record), assetsDigest: "0".repeat(64) } } }).where(eq(heartbeatRuns.id, f.runId)); + [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + } + release(); + if (invalid) await expect(late).rejects.toThrow("unverified"); + else expect((await late).receiptDigest).toBe(winner.receiptDigest); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(published); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_harness_filesystem_independence_verified")))).toHaveLength(1); + } finally { release(); await late.catch(() => undefined); } + }); + it("archives the successful source's raw controller and supervisor without changing execution authority", async () => { + const f = await controllerFixture(); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + const checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: f.input.companyId, issueId: f.issueId, runId: f.runId }); + const captured = captureLegacyDurablePrpCheckpoint(f.raw, { runId: f.runId, runnerInstanceId: before!.runnerInstanceId!, + normalizedSessionId: before!.nativeSessionId! }, { version: checkpoint!.runnerVersion, digest: checkpoint!.runnerDigest }); + expect([...captured.records].map(([key, value]) => [key, JSON.parse(value)])).toEqual( + [...checkpoint!.records].map(([key, value]) => [key, JSON.parse(value)])); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + const rows = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + expect(rows).toHaveLength(1); expect(rows[0]!.scopeDigest).not.toBe(f.input.scopeDigest); + expect(rows[0]!.status).toBe("current"); + const restored = join(f.mount, "isolated-raw-evidence"); + expect(await restoreDurableNativeHarnessBackup({ db, storage: f.storage, companyId: f.input.companyId, + scopeDigest: rows[0]!.scopeDigest, backupRoot: restored })).toBe(true); + expect(await readFile(join(restored, "current/control-plane-state.json"), "utf8")).toBe(f.raw); + expect(await readFile(join(restored, "current/supervisor.json"), "utf8")).toBe(f.supervisor); + expect(JSON.parse(await readFile(join(restored, "current/manifest.json"), "utf8"))) + .toMatchObject({ schema: "paperclip.legacy-controller-archive.v1", reason: "original_volume_source_evidence" }); + expect(await readFile(f.controllerPath, "utf8")).toBe(f.raw); + expect(await readFile(f.supervisorPath, "utf8")).toBe(f.supervisor); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(after).toEqual({ ...before, runnerProfileJson: { ...before!.runnerProfileJson, + nativeHarnessFilesystemIndependence: after!.runnerProfileJson!.nativeHarnessFilesystemIndependence } }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(owner); + expect(await hasLocalNativeHarnessObligations(db)).toBe(false); + }); + it.each(["codex-home", "opencode", "acpx"])("retains omitted %s history after switching to a preconfigured home", async directory => { + const f = await fixture("none"); + const execution = parseNativeExecutionInput({ ...f.execution, schema: "paperclip.native-execution-input.v7", homeMode: "preconfigured", + provider: { kind: "codex", model: null, approvalPolicy: "never" } }); + expect(scopedRunnerdStateRoot(execution)).toBe(f.sourceRoot); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...before!.runnerProfileJson, + nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, f.runId)); + const history = join(f.sourceRoot, directory, "history.json"); + await mkdir(join(history, ".."), { recursive: true, mode: 0o700 }); + await writeFile(history, "original provider history", { mode: 0o600 }); + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow("unverified"); + expect(await readFile(history, "utf8")).toBe("original provider history"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + expect(await hasLocalNativeHarnessObligations(db)).toBe(true); + }); + it("permits a preconfigured home only when no omitted provider directory remains", async () => { + const f = await fixture("none"); + const execution = parseNativeExecutionInput({ ...f.execution, schema: "paperclip.native-execution-input.v7", homeMode: "preconfigured", + provider: { kind: "codex", model: null, approvalPolicy: "never" } }); + expect(scopedRunnerdStateRoot(execution)).toBe(f.sourceRoot); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...before!.runnerProfileJson, + nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, f.runId)); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + expect(await hasLocalNativeHarnessObligations(db)).toBe(false); + }); + it("retains the first authenticated controller archive when a capture uploads late", async () => { + const f = await controllerFixture(); + let release!: () => void, entered!: () => void, delay = true; + const blocked = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { entered = resolve; }); + const put = f.storage.putFile.bind(f.storage); + vi.spyOn(f.storage, "putFile").mockImplementation(async args => { + const result = await put(args); + if (delay) { delay = false; entered(); await blocked; } + return result; + }); + const late = preserveNativeHarnessFilesystemIndependence(db, f.input); + try { + await ready; + const winner = await preserveNativeHarnessFilesystemIndependence(db, f.input); + const [published] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [archive] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, f.input.companyId), + eq(nativeHarnessBackups.status, "current"))); + release(); + expect((await late).receiptDigest).toBe(winner.receiptDigest); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(published); + expect(await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, f.input.companyId), + eq(nativeHarnessBackups.status, "current")))).toEqual([archive]); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.input.companyId), + eq(activityLog.action, "native_controller_source_evidence_preserved")))).toHaveLength(1); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + } finally { release(); await late.catch(() => undefined); } + }); + it.each(["checkpoint", "archive"])("invalidates preserved raw coverage when its %s binding changes", async binding => { + const f = await controllerFixture(); + await preserveNativeHarnessFilesystemIndependence(db, f.input); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(true); + if (binding === "checkpoint") await db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)); + else await db.update(nativeHarnessBackups).set({ status: "retired" }).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + expect(await hasValidNativeHarnessFilesystemIndependence(db, f.input)).toBe(false); + expect(await hasLocalNativeHarnessObligations(db)).toBe(true); + }); + it.each(["missing-checkpoint", "foreign-controller", "active-source", "changed-after-upload"])("holds raw controller evidence for %s", async control => { + const f = await controllerFixture(); + if (control === "missing-checkpoint") await db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)); + if (control === "foreign-controller") { const raw = JSON.parse(f.raw); raw.identity.runId = randomUUID(); await writeFile(f.controllerPath, JSON.stringify(raw)); } + if (control === "active-source") await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId)); + if (control === "changed-after-upload") { const put = f.storage.putFile.bind(f.storage); vi.spyOn(f.storage, "putFile").mockImplementation(async args => { + const result = await put(args); await writeFile(f.controllerPath, f.raw + "\n"); return result; + }); } + await expect(preserveNativeHarnessFilesystemIndependence(db, f.input)).rejects.toThrow(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) + .not.toHaveProperty("nativeHarnessFilesystemIndependence"); + const rows = await db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.companyId, f.input.companyId)); + expect(rows.every(row => row.status === "pending")).toBe(true); + }); +}); diff --git a/server/src/services/native-runtime/native-harness-filesystem-independence.ts b/server/src/services/native-runtime/native-harness-filesystem-independence.ts new file mode 100644 index 00000000000..e2c2eff9284 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-filesystem-independence.ts @@ -0,0 +1,427 @@ +import { constants } from "node:fs"; +import { lstat, open, readdir } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { basename, join, posix } from "node:path"; +import { and, asc, eq, inArray, or, sql } from "drizzle-orm"; +import { agentTaskSessions, companySkillRepositorySnapshots, companySkillVersions, environmentLeases, + heartbeatRuns, issues, nativeHarnessBackups, nativeRunnerCheckpoints, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; +import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "../remote-execution-termination.js"; +import { verifyFileManifest, type FileManifestEntry } from "../file-manifest-cache.js"; +import { skillPackageStore } from "../skill-package-store.js"; +import { skillPackageManifest } from "../skill-package-files.js"; +import { assertSkillSnapshotPath, skillFileBytes } from "../skill-snapshot.js"; +import { REPOSITORY_CONTENT_DIR, repositorySkillProjection } from "../skill-repository-package.js"; +import { captureLegacyDurablePrpCheckpoint, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { nativeSha256 } from "./canonical.js"; +import { readRuntimeSkillPreservation } from "./runtime-skill-preservation-reference.js"; +import { findNativeSessionResumeRun } from "./native-session-resume.js"; +import { resolveNativeHarnessPersistenceProfile, scopedRunnerdStateRoot } from "./native-session-executor.js"; +import { readNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { tryLockNativeWorkspaceFinalization } from "./native-workspace-finalization-ownership.js"; +import { assertOriginalHarnessVolumeSource, type OriginalHarnessVolumeCapability } from "./native-harness-original-volume.js"; +import { isKnownNativeSessionRootEntry } from "./native-session-root-entries.js"; +import { readLegacyNativeControllerArchiveSource, readLegacyNativeControllerArchiveDigests } from "./native-controller-migration.js"; +import { archiveNativeControllerSourceEvidence, legacyNativeControllerArchive } from "./native-controller-archive.js"; +import { nativeControllerArchiveCoordinatorDigest, nativeControllerArchiveProfileDigest, type SettledControllerArchiveOwner } from "./native-controller-archive-source.js"; +import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { readNativeControllerRecovery } from "./native-controller-recovery.js"; +import { verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; + +const KEY = "nativeHarnessFilesystemIndependence"; +const SCHEMA = "paperclip.native-harness-filesystem-independence.v1"; +const terminal = ["succeeded", "failed", "cancelled", "interrupted", "timed_out"]; +const sha = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +const digest = (value: unknown) => nativeSha256(JSON.parse(JSON.stringify(value))); +function fail(): never { throw new Error("native_harness_filesystem_independence_unverified"); } +const profile = (value: Record | null) => { const copy = { ...value }; delete copy[KEY]; return copy; }; +type PackageBinding = { id: string; digest: string }; +type VersionBinding = { id: string; digest: string }; +type ControllerBinding = { checkpointPartsDigest: string; archive: null | { id: string; digest: string; + controllerDigest: string; supervisorDigest: string | null } }; +type Receipt = { schema: typeof SCHEMA; selectedSessionId: string; lastRunId: string; sourceRunId: string; scopeDigest: string; + factsDigest: string; sourceCapabilityFingerprint: string; localFactsDigest: string; assetsDigest: string; + packages: PackageBinding[]; versions: VersionBinding[]; controller: ControllerBinding; observedAt: string }; +type Scope = { companyId: string; selectedSessionId: string; scopeDigest: string }; +type Run = typeof heartbeatRuns.$inferSelect; +const evidenceOwners = new WeakMap(); + +async function databaseFacts(db: Db, input: Scope, lock: boolean) { + const [selection] = await db.select().from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, input.companyId), + eq(agentTaskSessions.id, input.selectedSessionId), eq(agentTaskSessions.adapterType, "paperclip_runner"))); + if (!selection?.lastRunId || !/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail(); + const [anchor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, selection.agentId), eq(heartbeatRuns.id, selection.lastRunId))); + if (!anchor?.nativeIssueId || !anchor.nativeSessionId || anchor.runtimeMode !== "native") fail(); + const runsQuery = db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, selection.agentId), eq(heartbeatRuns.nativeIssueId, anchor.nativeIssueId), + eq(heartbeatRuns.nativeSessionId, anchor.nativeSessionId))).orderBy(asc(heartbeatRuns.id)).limit(501); + const runs = await runsQuery; + if (!runs.length || runs.length > 500) fail(); + if (lock) for (const run of runs) if (!await tryLockNativeWorkspaceFinalization(db, { companyId: input.companyId, runId: run.id })) fail(); + const ownersQuery = db.select().from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, input.companyId), + inArray(nativeRunFinalizations.runId, runs.map(run => run.id)))).orderBy(asc(nativeRunFinalizations.runId)); + const owners = await (lock ? ownersQuery.for("update") : ownersQuery); + const taskQuery = db.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, anchor.nativeIssueId))); + const [task] = await (lock ? taskQuery.for("update") : taskQuery); + if (!task || task.executionRunId || task.checkoutRunId) fail(); + if (lock) { + const [current] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, selection.id)).for("share"); + if (digest(current) !== digest(selection)) fail(); + const locked = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, runs.map(run => run.id))).orderBy(asc(heartbeatRuns.id)).for("share"); + if (digest(locked) !== digest(runs)) fail(); + } + const [clock] = await db.execute<{ now: Date }>(sql`select clock_timestamp() as now`); + const now = new Date(clock.now); + for (const run of runs) { + const owner = owners.find(row => row.runId === run.id); + if (!terminal.includes(run.status) || !run.finishedAt || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > now + || adapterExecutionControls.has(run.id) || run.runnerProfileJson?.nativeWorkspaceFinalizationOwner != null + || !owner || owner.leaseOwner !== null || owner.nextAttemptAt !== null + || !(owner.phase === "terminal_failure" || owner.phase === "committed" && owner.resultId)) fail(); + } + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + inArray(heartbeatRuns.status, ["queued", "running"]), or(eq(heartbeatRuns.nativeIssueId, anchor.nativeIssueId), + eq(heartbeatRuns.issueId, anchor.nativeIssueId), eq(heartbeatRuns.nativeSessionId, anchor.nativeSessionId)))).limit(1); + if (active) fail(); + const source = await findNativeSessionResumeRun(db, { companyId: input.companyId, agentId: anchor.agentId, + issueId: anchor.nativeIssueId, normalizedSessionId: anchor.nativeSessionId, beforeCreatedAt: anchor.createdAt, includeProgressBarrier: true }); + if (!source) fail(); + const execution = parseNativeExecutionInput(anchor.runnerProfileJson?.nativeExecutionInput); + const checkpoint = source.runnerProfileJson.sessionCheckpoint as { identity?: Record } | null; + if (!["codex", "opencode"].includes(execution.provider.kind) || !("runtimeContext" in execution) + || execution.binding.companyId !== anchor.companyId || execution.binding.agentId !== anchor.agentId + || execution.binding.issueId !== anchor.nativeIssueId || execution.binding.runId !== anchor.id + || execution.session.normalizedSessionId !== anchor.nativeSessionId || basename(scopedRunnerdStateRoot(execution)) !== input.scopeDigest + || !checkpoint || checkpoint.identity?.runId !== source.id || checkpoint.identity?.companyId !== input.companyId + || checkpoint.identity?.agentId !== anchor.agentId || checkpoint.identity?.issueId !== anchor.nativeIssueId + || checkpoint.identity?.sessionId !== anchor.nativeSessionId) fail(); + const leasesQuery = db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), + or(inArray(environmentLeases.heartbeatRunId, runs.map(run => run.id)), and(eq(environmentLeases.issueId, anchor.nativeIssueId), + sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'normalizedSessionId' = ${anchor.nativeSessionId}`)))) + .orderBy(asc(environmentLeases.id)); + const leases = await (lock ? leasesQuery.for("share") : leasesQuery); + if (!leases.length) fail(); + let matchingStamp = false; + const terminations = []; + for (const lease of leases) { + const resolved = await resolveRemoteTerminationLease(db, lease); + const receipt = resolved.metadata?.remoteExecutionTermination as { state?: unknown } | undefined; + if (!hasRemoteTerminationReceipt(resolved) || receipt?.state !== "destroyed") fail(); + terminations.push({ leaseId: lease.id, receipt }); + const stamp = readNativeHarnessBackupStamp(lease.metadata?.nativeHarnessBackup, lease.providerLeaseId ?? ""); + if (stamp?.sessionScopeSha256 === input.scopeDigest && stamp.normalizedSessionId === anchor.nativeSessionId + && runs.some(run => run.runnerInstanceId === stamp.runnerInstanceId) + && leases.some(row => row.providerLeaseId === stamp.sourceProviderLeaseId)) matchingStamp = true; + } + if (!matchingStamp) fail(); + const factsDigest = digest({ selection, runs: runs.map(run => ({ ...run, runnerProfileJson: profile(run.runnerProfileJson) })), owners, + task: { id: task.id, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId }, leases, terminations, + sourceRunId: source.id, sourceProfile: source.runnerProfileJson }); + return { anchor, execution, source, factsDigest }; +} + +async function assetBindings(db: Db, anchor: Run, sourceRunId: string) { + const preservation = readRuntimeSkillPreservation(anchor); + const versions: Array = [], packages = new Map(); + const sources = [anchor]; + if (sourceRunId !== anchor.id) { + const [source] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, anchor.companyId), eq(heartbeatRuns.id, sourceRunId))); + if (!source) fail(); sources.push(source); + } + for (const source of sources) { + const execution = parseNativeExecutionInput(source.runnerProfileJson?.nativeExecutionInput); + if (!("runtimeContext" in execution) || execution.binding.companyId !== source.companyId + || execution.binding.agentId !== source.agentId || execution.binding.issueId !== source.nativeIssueId + || execution.binding.runId !== source.id || execution.session.normalizedSessionId !== source.nativeSessionId) fail(); + const sourcePreservation = readRuntimeSkillPreservation(source); + for (const skill of execution.runtimeContext.skills) { + if (skill.versionId) { + const [version] = await db.select().from(companySkillVersions).where(and(eq(companySkillVersions.companyId, anchor.companyId), eq(companySkillVersions.id, skill.versionId))); + if (!version) fail(); + if (!versions.some(row => row.id === version.id)) versions.push(version); + const id = version.repositorySnapshotId ?? version.packageSnapshotId; + if (!id) continue; + const [row] = await db.select().from(companySkillRepositorySnapshots).where(and(eq(companySkillRepositorySnapshots.companyId, anchor.companyId), eq(companySkillRepositorySnapshots.id, id))); + if (!row) fail(); packages.set(id, row); + } else { + const asset = sourcePreservation?.assets.find(asset => asset.key === skill.key && asset.runtimeName === skill.runtimeName); + if (!asset) fail(); + runtimeFiles(skill, asset.manifestText); + const [row] = await db.select().from(companySkillRepositorySnapshots).where(and(eq(companySkillRepositorySnapshots.companyId, anchor.companyId), eq(companySkillRepositorySnapshots.id, asset.packageSnapshotId))); + if (!row || row.contentHash !== asset.packageContentHash) fail(); packages.set(row.id, row); + } + } + } + for (const row of packages.values()) if (row.storage?.provider !== "s3" || row.storage.durable !== true) fail(); + return { versions, packages, preservation, + bindings: { versions: versions.map(row => ({ id: row.id, digest: digest(row) })).sort((a, b) => a.id.localeCompare(b.id)), + packages: [...packages.values()].map(row => ({ id: row.id, digest: digest({ id: row.id, companyId: row.companyId, + contentHash: row.contentHash, storage: row.storage }) })).sort((a, b) => a.id.localeCompare(b.id)) } }; +} + +function matchingReceipt(value: unknown, facts: Awaited>, input: Scope, + bindings: Awaited>["bindings"], controller: ControllerBinding): Receipt | null { + const receipt = value as Receipt | undefined; + if (receipt?.schema !== SCHEMA || receipt.selectedSessionId !== input.selectedSessionId || receipt.scopeDigest !== input.scopeDigest + || receipt.lastRunId !== facts.anchor.id || receipt.sourceRunId !== facts.source.id || receipt.factsDigest !== facts.factsDigest + || !/^[a-f0-9]{64}$/.test(receipt.sourceCapabilityFingerprint) || !/^[a-f0-9]{64}$/.test(receipt.localFactsDigest) + || !Number.isFinite(Date.parse(receipt.observedAt)) || !Array.isArray(receipt.packages) || !Array.isArray(receipt.versions)) return null; + return digest(bindings) === receipt.assetsDigest && digest(receipt.packages) === digest(bindings.packages) + && digest(receipt.versions) === digest(bindings.versions) && receipt.controller != null + && digest(receipt.controller) === digest(controller) ? receipt : null; +} + +export async function hasValidNativeHarnessFilesystemIndependence(db: Db, input: Scope): Promise { + try { + const facts = await databaseFacts(db, input, false); + const assets = await assetBindings(db, facts.anchor, facts.source.id); + const existing = facts.anchor.runnerProfileJson?.[KEY] as Receipt | undefined; + const controller = await controllerBinding(db, facts, existing?.controller?.archive ?? null, false); + return matchingReceipt(existing, facts, input, assets.bindings, controller) !== null; + } catch { return false; } +} + +export async function assertNativeHarnessFilesystemArchiveSource(db: Db, owner: SettledControllerArchiveOwner) { + const registered = evidenceOwners.get(owner); + if (!registered || !owner.filesystemIndependence) fail(); + const facts = await databaseFacts(db, registered.scope, true); + if (facts.factsDigest !== registered.factsDigest || facts.factsDigest !== owner.filesystemIndependence.factsDigest + || facts.source.id !== owner.runId || facts.anchor.id !== owner.selectedLastRunId) fail(); +} + +async function controllerBinding(db: Db, facts: Awaited>, archive: ControllerBinding["archive"], lock: boolean): Promise { + const partsQuery = db.select().from(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.companyId, facts.anchor.companyId), + eq(nativeRunnerCheckpoints.issueId, facts.anchor.nativeIssueId!), eq(nativeRunnerCheckpoints.runId, facts.source.id))) + .orderBy(asc(nativeRunnerCheckpoints.part)); + const parts = await (lock ? partsQuery.for("share") : partsQuery); + if (archive) { + if (!parts.some(row => row.part === "header") || !readNativeControllerRecovery( + facts.source.runnerProfileJson.nativeControllerRecovery, facts.source.runnerProfileJson.sessionCheckpoint)) fail(); + const rowQuery = db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, facts.anchor.companyId), + eq(nativeHarnessBackups.runId, facts.source.id), eq(nativeHarnessBackups.id, archive.id))); + const [row] = await (lock ? rowQuery.for("share") : rowQuery); + if (!row || !["current", "superseded"].includes(row.status) || digest(row) !== archive.digest) fail(); + } + return { checkpointPartsDigest: digest(parts), archive }; +} + +async function preserveControllerEvidence(db: Db, facts: Awaited>, input: Scope & { + source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; +}): Promise { + const material = await readLegacyNativeControllerArchiveSource(facts.source); + if (material.paths.root !== scopedRunnerdStateRoot(facts.execution)) fail(); + if (material.controller === undefined) { + if (material.supervisor !== undefined) fail(); + return controllerBinding(db, facts, null, false); + } + const identity = JSON.parse(material.controller).identity as Record | undefined; + if (identity?.runId !== facts.source.id || identity.runnerInstanceId !== facts.source.runnerInstanceId + || identity.normalizedSessionId !== facts.anchor.nativeSessionId) fail(); + const checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: facts.anchor.nativeIssueId!, runId: facts.source.id }); + if (!checkpoint || !readNativeControllerRecovery(facts.source.runnerProfileJson.nativeControllerRecovery, + facts.source.runnerProfileJson.sessionCheckpoint)) fail(); + const captured = captureLegacyDurablePrpCheckpoint(material.controller, { runId: facts.source.id, + runnerInstanceId: facts.source.runnerInstanceId!, normalizedSessionId: facts.anchor.nativeSessionId! }, + { version: checkpoint.runnerVersion, digest: checkpoint.runnerDigest }); + if (captured.records.size !== checkpoint.records.size || [...captured.records].some(([key, value]) => { + const durable = checkpoint.records.get(key); return durable === undefined || digest(JSON.parse(value)) !== digest(JSON.parse(durable)); + })) fail(); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.runId, facts.source.id))); + if (!coordinator) fail(); + const owner: SettledControllerArchiveOwner = Object.freeze({ kind: "settled_controller_archive", companyId: input.companyId, + issueId: facts.anchor.nativeIssueId!, agentId: facts.anchor.agentId, runId: facts.source.id, sessionId: facts.anchor.nativeSessionId!, + selectedSessionId: input.selectedSessionId, selectedLastRunId: facts.anchor.id, + profileDigest: nativeControllerArchiveProfileDigest(facts.source.runnerProfileJson), coordinatorDigest: nativeControllerArchiveCoordinatorDigest(coordinator), + controllerDigest: sha(material.controller), supervisorDigest: material.supervisor === undefined ? null : sha(material.supervisor), + filesystemIndependence: Object.freeze({ source: input.source, scopeDigest: input.scopeDigest, factsDigest: facts.factsDigest }) }); + evidenceOwners.set(owner, { scope: { companyId: input.companyId, selectedSessionId: input.selectedSessionId, scopeDigest: input.scopeDigest }, factsDigest: facts.factsDigest }); + const source = { companyId: input.companyId, issueId: facts.anchor.nativeIssueId!, runId: facts.source.id, + controller: material.controller, supervisor: material.supervisor, purpose: "filesystem_independence" as const }; + await archiveNativeControllerSourceEvidence(db, { ...source, onlineOwner: owner, storage: input.storage, signal: input.signal }); + const descriptor = legacyNativeControllerArchive(source); + const [row] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, input.companyId), + eq(nativeHarnessBackups.runId, facts.source.id), eq(nativeHarnessBackups.scopeDigest, descriptor.scopeDigest), + eq(nativeHarnessBackups.manifestSha256, descriptor.manifestSha256), eq(nativeHarnessBackups.status, "current"))); + if (!row) fail(); + if (!await verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: input.companyId, + scopeDigest: descriptor.scopeDigest, manifestSha256: descriptor.manifestSha256, backupId: row.id, signal: input.signal })) fail(); + return controllerBinding(db, facts, { id: row.id, digest: digest(row), controllerDigest: owner.controllerDigest, + supervisorDigest: owner.supervisorDigest }, false); +} + +function runtimeFiles(skill: { bundle: { digest: string; manifestDigest: string; fileCount: number; totalBytes: number } }, + text: string): FileManifestEntry[] { + if (Buffer.byteLength(text) > 4 * 1024 * 1024 || sha(text) !== skill.bundle.manifestDigest) fail(); + const manifest = JSON.parse(text); + if (manifest.schema !== "paperclip.runtime-asset-manifest.v1" || manifest.digest !== skill.bundle.digest + || manifest.fileCount !== skill.bundle.fileCount || manifest.totalBytes !== skill.bundle.totalBytes || !Array.isArray(manifest.files) + || manifest.files.length !== manifest.fileCount || manifest.files.length > 10_000 || sha(JSON.stringify(manifest.files)) !== manifest.digest) fail(); + let total = 0; const paths = new Set(); + const files = manifest.files.map((file: { path: string; size: number; sha256: string; mode: number }) => { + assertSkillSnapshotPath(file.path); + if (paths.has(file.path) || !Number.isSafeInteger(file.size) || file.size < 0 || !/^[a-f0-9]{64}$/.test(file.sha256) + || !Number.isInteger(file.mode) || (file.mode & ~0o555) !== 0) fail(); + paths.add(file.path); total += file.size; + if (total > skill.bundle.totalBytes) fail(); + return { ...file, executable: Boolean(file.mode & 0o111) }; + }); + if (total !== skill.bundle.totalBytes) fail(); + return files; +} + +async function localFacts(facts: Awaited>, assets: Awaited>, + storage: StorageService, db: Db, verifyStorage: boolean, signal: AbortSignal, controller: ControllerBinding, plans = new Map()) { + const root = scopedRunnerdStateRoot(facts.execution), metadata: unknown[] = [], store = skillPackageStore(db, storage); + const directories = resolveNativeHarnessPersistenceProfile(facts.execution).directories; + const stat = async (path: string) => lstat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + const rootStat = await stat(root); + if (rootStat) { + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) fail(); + const names = await readdir(root); + if (names.some(name => !isKnownNativeSessionRootEntry(name))) fail(); + if (names.some(name => ["runner", "codex-home", "opencode", "acpx"].includes(name) + && !directories.some(directory => directory.name === name))) fail(); + const control = await stat(join(root, "control-plane")); + if (control && (!control.isDirectory() || control.isSymbolicLink() + || (await readdir(join(root, "control-plane"))).some(name => name !== "control-plane-state.json"))) fail(); + for (const name of ["continuity-breaks", "authority-epochs", "provider-artifacts", "failover-backups"]) { + const entry = await stat(join(root, name)); + if (entry && (!entry.isDirectory() || entry.isSymbolicLink() || (await readdir(join(root, name))).length)) fail(); + } + } + const material = await readLegacyNativeControllerArchiveDigests(facts.source); + if (controller.archive ? material.controllerDigest !== controller.archive.controllerDigest + || (material.supervisorDigest ?? null) !== controller.archive.supervisorDigest + : material.controllerDigest !== undefined || material.supervisorDigest !== undefined) fail(); + metadata.push({ controller: material.controllerDigest ?? null, supervisor: material.supervisorDigest ?? null }); + for (const candidate of ["current", "previous"]) if (await stat(join(root, "failover-backups", candidate))) fail(); + if (verifyStorage) for (const row of assets.packages.values()) if (!await store.verifyDurable(row.companyId, row.id, signal)) fail(); + const runtime = "runtimeContext" in facts.execution ? facts.execution.runtimeContext : null; + if (!runtime) fail(); + let bytes = 0, count = 0; + for (const directory of directories) { + const path = join(root, directory.name), directoryStat = await stat(path); + metadata.push([directory.name, directoryStat && [directoryStat.dev, directoryStat.ino, directoryStat.ctimeMs]]); + if (!directoryStat) continue; + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) fail(); + const walk = async (current: string, relative: string): Promise => { + for (const name of (await readdir(current)).sort()) { + signal.throwIfAborted(); if (++count > 25_000) fail(); + const rel = relative ? `${relative}/${name}` : name, file = join(current, name), s = await lstat(file); + if (s.isSymbolicLink() || !s.isFile() && !s.isDirectory()) fail(); + metadata.push([directory.name, rel, s.dev, s.ino, s.size, s.mode, s.mtimeMs, s.ctimeMs]); + if (directory.excludeEntries.some(excluded => rel === excluded || rel.startsWith(`${excluded}/`))) continue; + if (directory.name !== "codex-home" || rel !== "skills" && !rel.startsWith("skills/")) fail(); + if (s.isDirectory()) await walk(file, rel); + } + }; + await walk(path, ""); + if (directory.name !== "codex-home") continue; + const skillRoot = join(path, "skills"); + if (!await stat(skillRoot)) continue; + for (const name of await readdir(skillRoot)) { + const skill = runtime.skills.find(skill => skill.runtimeName === name); + if (!skill || !/^[A-Za-z0-9._-]+$/.test(name)) fail(); + const projected = join(skillRoot, name); let files: FileManifestEntry[]; + if (!verifyStorage) { + const captured = plans.get(projected); if (!captured) fail(); files = captured; + } else if (!skill.versionId) { + const preserved = assets.preservation?.assets.find(asset => asset.key === skill.key && asset.runtimeName === name); + if (!preserved) fail(); files = runtimeFiles(skill, preserved.manifestText); + const manifest = await store.manifest(facts.anchor.companyId, preserved.packageSnapshotId, signal); + if (manifest.files.length !== files.length || files.some(file => !manifest.files.some(actual => actual.path === file.path + && actual.sha256 === file.sha256 && actual.size === file.size && actual.executable === file.executable))) fail(); + } else { + const version = assets.versions.find(version => version.id === skill.versionId); if (!version) fail(); + const id = version.repositorySnapshotId ?? version.packageSnapshotId; + const manifest = id ? await store.manifest(facts.anchor.companyId, id, signal) : skillPackageManifest(version.fileInventory); + let projectedFiles = manifest.files; + if (version.repositorySnapshotId) { + if (!version.repositorySkillPath) fail(); + const wanted = new Set([version.repositorySkillPath, posix.join(posix.dirname(version.repositorySkillPath), "agents/openai.yaml")]); + const authored = []; + for (const file of manifest.files.filter(file => wanted.has(file.path))) { + const handle = await open(join(projected, REPOSITORY_CONTENT_DIR, file.path), constants.O_RDONLY | constants.O_NOFOLLOW); + try { const s = await handle.stat(); if (!s.isFile() || s.size !== file.size || s.size > 4 * 1024 * 1024) fail(); + const bytes = await handle.readFile(); if (sha(bytes) !== file.sha256) fail(); + authored.push({ ...file, content: bytes.toString(file.encoding) }); + } finally { await handle.close(); } + } + projectedFiles = [...manifest.files.map(file => ({ ...file, path: `${REPOSITORY_CONTENT_DIR}/${file.path}` })), + ...repositorySkillProjection(authored, version.repositorySkillPath).filter(file => !file.path.startsWith(`${REPOSITORY_CONTENT_DIR}/`)) + .map(file => { const bytes = skillFileBytes(file); return { ...file, size: bytes.length, sha256: sha(bytes), executable: Boolean(file.executable), encoding: file.encoding ?? "utf8" as const }; })]; + } else if (version.packageFileMap) projectedFiles = version.packageFileMap.map(mapping => { + const file = manifest.files.find(file => file.path === mapping.sourcePath); if (!file) fail(); return { ...file, path: mapping.path }; + }); + files = projectedFiles.map(file => ({ path: file.path, size: file.size, sha256: file.sha256, executable: file.executable, mode: file.executable ? 0o555 : 0o444 })); + const runtimeManifest = files.map(({ path, size, sha256, mode }) => ({ path, sha256, mode, size })).sort((a, b) => a.path.localeCompare(b.path)); + if (sha(JSON.stringify(runtimeManifest)) !== skill.bundle.digest || files.length !== skill.bundle.fileCount + || files.reduce((n, file) => n + file.size, 0) !== skill.bundle.totalBytes) fail(); + runtimeFiles(skill, JSON.stringify({ schema: "paperclip.runtime-asset-manifest.v1", digest: skill.bundle.digest, + fileCount: files.length, totalBytes: skill.bundle.totalBytes, files: runtimeManifest }) + "\n"); + } + bytes += skill.bundle.totalBytes; if (bytes > 64 * 1024 * 1024) fail(); + if (!await verifyFileManifest(projected, files)) fail(); + plans.set(projected, files); + } + } + return { digest: digest(metadata), plans }; +} + +export async function preserveNativeHarnessFilesystemIndependence(db: Db, input: Scope & { + source: OriginalHarnessVolumeCapability; storage: StorageService; signal: AbortSignal; +}) { + await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); + if (input.storage.provider !== "s3") fail(); + const facts = await databaseFacts(db, input, false), assets = await assetBindings(db, facts.anchor, facts.source.id); + const controller = await preserveControllerEvidence(db, facts, input); + const local = await localFacts(facts, assets, input.storage, db, true, input.signal, controller), localFactsDigest = local.digest; + const previous = facts.anchor.runnerProfileJson?.[KEY]; + if (previous != null) { + if (!await hasValidNativeHarnessFilesystemIndependence(db, input)) fail(); + return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(previous) }; + } + const receipt: Receipt = { schema: SCHEMA, selectedSessionId: input.selectedSessionId, scopeDigest: input.scopeDigest, + lastRunId: facts.anchor.id, sourceRunId: facts.source.id, factsDigest: facts.factsDigest, + sourceCapabilityFingerprint: input.source.fingerprint, localFactsDigest, assetsDigest: digest(assets.bindings), ...assets.bindings, controller, observedAt: new Date().toISOString() }; + let publishedReceipt = receipt; + const publications: ActivityPublication[] = []; + await db.transaction(async transaction => { + const tx = transaction as unknown as Db; + await tx.execute(sql`set local lock_timeout = '1s'`); + const current = await databaseFacts(tx, input, true); + if (current.factsDigest !== facts.factsDigest) fail(); + for (const row of [...assets.versions].sort((a, b) => a.id.localeCompare(b.id))) { + await tx.select({ id: companySkillVersions.id }).from(companySkillVersions).where(eq(companySkillVersions.id, row.id)).for("share"); + } + for (const row of [...assets.packages.values()].sort((a, b) => a.id.localeCompare(b.id))) { + await tx.select({ id: companySkillRepositorySnapshots.id }).from(companySkillRepositorySnapshots) + .where(eq(companySkillRepositorySnapshots.id, row.id)).for("update"); + } + const currentAssets = await assetBindings(tx, current.anchor, current.source.id); + const currentController = await controllerBinding(tx, current, controller.archive, true); + if (digest(currentAssets.bindings) !== receipt.assetsDigest + || digest(currentController) !== digest(controller) + || (await localFacts(current, currentAssets, input.storage, tx, false, input.signal, controller, local.plans)).digest !== localFactsDigest) fail(); + await assertOriginalHarnessVolumeSource(input.source); input.signal.throwIfAborted(); + const existing = current.anchor.runnerProfileJson?.[KEY]; + if (existing != null) { + const winner = matchingReceipt(existing, current, input, currentAssets.bindings, currentController); + if (!winner) fail(); + publishedReceipt = winner; + return; + } + await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...current.anchor.runnerProfileJson, [KEY]: receipt } }) + .where(and(eq(heartbeatRuns.id, current.anchor.id), eq(heartbeatRuns.companyId, input.companyId))); + await logActivity(tx, { companyId: input.companyId, actorType: "system", actorId: "native-harness-preservation", + action: "native_harness_filesystem_independence_verified", entityType: "heartbeat_run", entityId: current.anchor.id, + agentId: current.anchor.agentId, runId: current.anchor.id, details: { scopeDigest: input.scopeDigest, + sourceRunId: current.source.id, sourceCapabilityFingerprint: receipt.sourceCapabilityFingerprint, factsDigest: receipt.factsDigest } }, publications); + }); + for (const publication of publications) publishActivity(publication); + return { scopeDigest: input.scopeDigest, sourceRunId: facts.source.id, receiptDigest: digest(publishedReceipt) }; +} diff --git a/server/src/services/native-runtime/native-harness-original-volume.test.ts b/server/src/services/native-runtime/native-harness-original-volume.test.ts new file mode 100644 index 00000000000..bf566212977 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-original-volume.test.ts @@ -0,0 +1,68 @@ +import { randomUUID, createHash } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { originalHarnessVolumeAnchorPath, verifyOriginalHarnessVolumeSource, assertOriginalHarnessVolumeSource, + type OriginalHarnessVolumeAnchor, type ReviewedOriginalHarnessVolumeWitness } from "./native-harness-original-volume.js"; + +vi.mock("../../build-commit.js", () => ({ readBuildCommit: () => "a".repeat(40) })); +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); vi.useRealTimers(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function fixture() { + const root = await realpath(await mkdtemp(join(tmpdir(), "harness-original-volume-"))); roots.push(root); + const projectId = randomUUID(), serviceId = randomUUID(), environmentId = randomUUID(), volumeId = randomUUID(), deploymentId = randomUUID(); + for (const [key, value] of Object.entries({ PAPERCLIP_HOME: root, PAPERCLIP_INSTANCE_ID: "default", RAILWAY_PROJECT_ID: projectId, + RAILWAY_SERVICE_ID: serviceId, RAILWAY_ENVIRONMENT_ID: environmentId, RAILWAY_VOLUME_ID: volumeId, + RAILWAY_DEPLOYMENT_ID: deploymentId, RAILWAY_VOLUME_MOUNT_PATH: root })) vi.stubEnv(key, value); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", join(root, "instances/default/runtime/paperclip-runner/durable-sessions")); + const anchorPath = originalHarnessVolumeAnchorPath(); await mkdir(join(root, "instances/default/recovery"), { recursive: true, mode: 0o700 }); + await mkdir(process.env.PAPERCLIP_RUNNER_STATE_DIR!, { recursive: true, mode: 0o700 }); + const m = await lstat(root), i = await lstat(join(root, "instances/default")); + const anchor: OriginalHarnessVolumeAnchor = { schema: "paperclip.native-harness-original-volume-anchor.v1", nonce: "b".repeat(64), + projectId, serviceId, environmentId, volumeId, volumeInstanceId: randomUUID(), mountPath: root, + mountIdentity: { dev: m.dev, ino: m.ino }, instanceIdentity: { dev: i.dev, ino: i.ino }, + createdFrom: { deploymentId: randomUUID(), commit: "c".repeat(40), providerWitnessSha256: "d".repeat(64) }, createdAt: new Date().toISOString() }; + const bytes = Buffer.from(JSON.stringify(anchor)); await writeFile(anchorPath, bytes, { mode: 0o600 }); + const witness: ReviewedOriginalHarnessVolumeWitness = { projectId, serviceId, environmentId, volumeId, volumeInstanceId: anchor.volumeInstanceId, + mountPath: root, currentDeploymentId: deploymentId, currentCommit: "a".repeat(40), currentMountIdentity: { dev: m.dev, ino: m.ino }, + currentInstanceIdentity: { dev: i.dev, ino: i.ino }, observedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 120_000).toISOString(), anchorSha256: createHash("sha256").update(bytes).digest("hex"), providerWitnessSha256: "e".repeat(64) }; + return { root, anchorPath, anchor, witness }; +} + +it("requires the positive original-volume anchor and permits a later normal deployment", async () => { + const { witness } = await fixture(); + const capability = await verifyOriginalHarnessVolumeSource(witness); + await expect(assertOriginalHarnessVolumeSource(capability)).resolves.toBeUndefined(); + expect(capability.fingerprint).toMatch(/^[a-f0-9]{64}$/); + await expect(assertOriginalHarnessVolumeSource({ fingerprint: capability.fingerprint } as never)).rejects.toThrow("native_harness_original_volume_required"); +}); +it.each(["expired", "future", "wrong-deployment", "missing-volume", "wrong-mount", "external-state", "changed-anchor", "public-anchor", "symlink-anchor"])("refuses %s", async control => { + const { root, witness, anchorPath } = await fixture(); + if (control === "expired") witness.observedAt = new Date(Date.now() - 120_001).toISOString(); + if (control === "future") witness.observedAt = new Date(Date.now() + 1000).toISOString(); + if (control === "wrong-deployment") vi.stubEnv("RAILWAY_DEPLOYMENT_ID", randomUUID()); + if (control === "missing-volume") vi.stubEnv("RAILWAY_VOLUME_ID", ""); + if (control === "wrong-mount") vi.stubEnv("RAILWAY_VOLUME_MOUNT_PATH", "/"); + if (control === "external-state") vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", "/tmp/outside-instance"); + if (control === "changed-anchor") await writeFile(anchorPath, "{}"); + if (control === "public-anchor") await chmod(anchorPath, 0o644); + if (control === "symlink-anchor") { await rm(anchorPath); const other = join(root, "alias.json"); await writeFile(other, "{}"); await symlink(other, anchorPath); } + await expect(verifyOriginalHarnessVolumeSource(witness)).rejects.toThrow(); +}); +it("revalidates the anchor and current deployment before publication", async () => { + const { witness, anchorPath } = await fixture(); + const capability = await verifyOriginalHarnessVolumeSource(witness); + await writeFile(anchorPath, "{}"); + await expect(assertOriginalHarnessVolumeSource(capability)).rejects.toThrow("native_harness_original_volume_required"); +}); +it("treats creation device/inode as provenance after a provider-confirmed remount", async () => { + const { witness, anchor, anchorPath } = await fixture(); + anchor.mountIdentity = { dev: 1, ino: 2 }; anchor.instanceIdentity = { dev: 1, ino: 3 }; + const bytes = Buffer.from(JSON.stringify(anchor)); await writeFile(anchorPath, bytes); + witness.anchorSha256 = createHash("sha256").update(bytes).digest("hex"); + await expect(verifyOriginalHarnessVolumeSource(witness)).resolves.toBeDefined(); + witness.currentMountIdentity.dev++; + await expect(verifyOriginalHarnessVolumeSource(witness)).rejects.toThrow("native_harness_original_volume_required"); +}); diff --git a/server/src/services/native-runtime/native-harness-original-volume.ts b/server/src/services/native-runtime/native-harness-original-volume.ts new file mode 100644 index 00000000000..f7cc5e91399 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-original-volume.ts @@ -0,0 +1,97 @@ +import { constants } from "node:fs"; +import { lstat, open, realpath } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { dirname, relative, resolve, sep } from "node:path"; +import { readBuildCommit } from "../../build-commit.js"; +import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; +import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; +import { nativeSha256 } from "./canonical.js"; + +type FileIdentity = { dev: number; ino: number }; +export type OriginalHarnessVolumeAnchor = { + schema: "paperclip.native-harness-original-volume-anchor.v1"; + nonce: string; + projectId: string; serviceId: string; environmentId: string; volumeId: string; volumeInstanceId: string; + mountPath: string; mountIdentity: FileIdentity; instanceIdentity: FileIdentity; + createdFrom: { deploymentId: string; commit: string; providerWitnessSha256: string }; + createdAt: string; +}; +export type ReviewedOriginalHarnessVolumeWitness = { + projectId: string; serviceId: string; environmentId: string; volumeId: string; volumeInstanceId: string; + mountPath: string; currentDeploymentId: string; currentCommit: string; + currentMountIdentity: FileIdentity; currentInstanceIdentity: FileIdentity; + observedAt: string; expiresAt: string; anchorSha256: string; providerWitnessSha256: string; +}; +const brand: unique symbol = Symbol("original native harness volume"); +export type OriginalHarnessVolumeCapability = { readonly [brand]: true; readonly fingerprint: string }; +const capabilities = new WeakMap(); +const sha = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +const reject = (): never => { throw new Error("native_harness_original_volume_required"); }; +const inside = (root: string, path: string) => { const rel = relative(root, path); return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !rel.startsWith(sep); }; +const uuid = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i; +const digest = /^[a-f0-9]{64}$/; + +export function originalHarnessVolumeAnchorPath() { + return resolve(resolvePaperclipInstanceRoot(), "recovery", "native-harness-original-volume-anchor.json"); +} + +async function verify(witness: ReviewedOriginalHarnessVolumeWitness) { + const now = Date.now(), observed = Date.parse(witness.observedAt), expiry = Date.parse(witness.expiresAt); + if (!Number.isFinite(observed) || !Number.isFinite(expiry) || observed > now || now - observed > 120_000 + || expiry <= now || expiry - observed > 120_000 || !digest.test(witness.anchorSha256) + || !digest.test(witness.providerWitnessSha256) || !/^[a-f0-9]{40}$/.test(witness.currentCommit)) reject(); + for (const [key, value] of Object.entries({ RAILWAY_PROJECT_ID: witness.projectId, RAILWAY_SERVICE_ID: witness.serviceId, + RAILWAY_ENVIRONMENT_ID: witness.environmentId, RAILWAY_VOLUME_ID: witness.volumeId, + RAILWAY_DEPLOYMENT_ID: witness.currentDeploymentId, RAILWAY_VOLUME_MOUNT_PATH: witness.mountPath })) { + if (!value || process.env[key] !== value) reject(); + } + if (readBuildCommit() !== witness.currentCommit || !uuid.test(witness.volumeInstanceId)) reject(); + const mount = resolve(witness.mountPath), instance = resolvePaperclipInstanceRoot(), state = nativeHarnessStateBase(); + if (!inside(mount, instance) || !inside(instance, state) || await realpath(mount) !== mount) reject(); + const anchorPath = originalHarnessVolumeAnchorPath(); + for (let path = dirname(anchorPath); path !== mount; path = dirname(path)) { + if (!inside(mount, path)) reject(); + const stat = await lstat(path); + if (!stat.isDirectory() || stat.isSymbolicLink() + || path === dirname(anchorPath) && ((stat.mode & 0o777) !== 0o700 || stat.uid !== process.getuid?.())) reject(); + } + const before = await lstat(anchorPath); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > 8192 + || (before.mode & 0o777) !== 0o600 || before.uid !== process.getuid?.()) reject(); + const handle = await open(anchorPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let anchor: OriginalHarnessVolumeAnchor; + try { + const opened = await handle.stat(); + if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) reject(); + const bytes = await handle.readFile(); + if (sha(bytes) !== witness.anchorSha256) reject(); + anchor = JSON.parse(bytes.toString("utf8")); + const after = await handle.stat(); + if (after.ctimeMs !== before.ctimeMs || after.mtimeMs !== before.mtimeMs || after.size !== before.size) reject(); + } finally { await handle.close(); } + if (anchor.schema !== "paperclip.native-harness-original-volume-anchor.v1" || !digest.test(anchor.nonce) + || !/^[a-f0-9]{40}$/.test(anchor.createdFrom?.commit) || !uuid.test(anchor.createdFrom?.deploymentId) + || !digest.test(anchor.createdFrom?.providerWitnessSha256) || !Number.isFinite(Date.parse(anchor.createdAt))) reject(); + for (const key of ["projectId", "serviceId", "environmentId", "volumeId", "volumeInstanceId", "mountPath"] as const) { + if (anchor[key] !== witness[key]) reject(); + } + const mountStat = await lstat(mount), instanceStat = await lstat(instance), stateStat = await lstat(state); + if (!mountStat.isDirectory() || mountStat.isSymbolicLink() || !instanceStat.isDirectory() || instanceStat.isSymbolicLink() + || mountStat.dev !== witness.currentMountIdentity?.dev || mountStat.ino !== witness.currentMountIdentity?.ino + || instanceStat.dev !== witness.currentInstanceIdentity?.dev || instanceStat.ino !== witness.currentInstanceIdentity?.ino + || !stateStat.isDirectory() || stateStat.isSymbolicLink() || await realpath(state) !== state + || before.dev !== mountStat.dev || instanceStat.dev !== mountStat.dev || stateStat.dev !== mountStat.dev) reject(); + return nativeSha256({ witness, anchorSha256: witness.anchorSha256 }); +} + +export async function verifyOriginalHarnessVolumeSource(witness: ReviewedOriginalHarnessVolumeWitness): Promise { + const frozen = structuredClone(witness), fingerprint = await verify(frozen); + const capability = Object.freeze({ [brand]: true as const, fingerprint }); + capabilities.set(capability, frozen); + return capability; +} + +export async function assertOriginalHarnessVolumeSource(capability: OriginalHarnessVolumeCapability) { + const witness = capabilities.get(capability); + if (!witness || await verify(witness) !== capability.fingerprint) reject(); +}