diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 4b2df094f3a..8bc8886b075 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1807,7 +1807,11 @@ experimental isolated-workspaces UI. Ordinary workspace updates remain gated; the runtime can bind only a validated shared workspace in the issue's company and project. Follow-ups can therefore reuse the same sandbox and provider session. A staged provider package is reused only after the complete expected -manifest and artifact hashes verify. A missing, changed, or incompatible package +manifest and artifact hashes verify. Exact live-process reattachment uses the +controller-owned pack and runner identities recorded after the original session +passed authentication. It preserves protocol and capability checks. Fresh or +replacement launches use current release authority. A missing historical receipt +retains the current release gate; guest files cannot establish old authority. A missing, changed, or incompatible package must be replaced and verified before launch. Warm attachment requires two consecutive authenticated readiness snapshots. diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index f6ba0ee73ea..8b9c004282c 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -371,6 +371,26 @@ and operator guidance remain under Recovery details. Source-only repair remains available through the board-authorized recovery resolution API and its original stop-receipt and evidence checks. +After a remote provider pack passes full integrity verification and the session +passes authenticated admission, the controller saves its manifest and runner +binary identity in `provider-pack-authority.json`. The receipt binds the company, +agent, workspace, session, runner instance, environment lease, remote working +directory, and provider descriptor. It stays in private controller state, outside +guest workspace and harness backup directories. Projectless runs use their +original environment lease as the workspace identity because each run otherwise +has a new workspace placeholder. Real workspace identities still require an exact +match. A receipt write failure warns and returns the admitted session to its +normal owner; it does not discard the live session handle. Future recovery still +fails closed without matching authority. + +Exact process reattachment uses that saved release authority. It still verifies +the full remote pack, runner protocol and capabilities, exact process identity, +and authenticated PRP handshake. A new controller release alone does not require +a live provider to change its executable files. Fresh launches and explicit +session replacement select current release artifacts. Invalid or mismatched +receipts fail closed. Older sessions without a receipt retain the strict current +release gate; a sandbox's own manifest cannot supply missing launch authority. + Remote provider-pack verification retries an interrupted read at most once. Each command retains its 30-second transport timeout. A timeout, signal, absent exit status, or transport exception does not prove artifact incompatibility. @@ -380,7 +400,9 @@ a new provider attempt, so it cannot renew this exhausted read budget. A completed nonzero verification, digest mismatch, or wrong OpenCode version remains incompatible and cannot authorize a new provider turn. Recovery retains the exact source workspace and existing ownership gates; -these read-only retries do not replay tools or alter a retained pack. +these read-only retries do not replay tools or alter a retained pack. Verification +errors include bounded, redacted output from both transport streams, including +providers that combine stderr into stdout. ### Continue after confirmed source destruction diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 6624fbc67f7..b7de12dddc4 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -176,6 +176,14 @@ and rejected identity evidence. The Runner restart tests require positive exit evidence before replacement. This coverage does not prove recovery of a historical retained sandbox or behavior during a live provider outage. +For a release change during a remote provider session, verify that reattachment +uses the controller's admitted pack manifest and runner digest. Check useful +output from the same process without replaying a turn. Repeat with a modified +sidecar, mismatched receipt scope, replacement lease, and incompatible runner +protocol. The [authority tests](../../server/src/services/native-runtime/remote-provider-pack-authority.test.ts) +and executor wiring regressions cover selection and fail-closed scope checks. +They do not prove a historical retained sandbox was recovered. + For completion recovery, restart after a native completion proposal is appended but before its session checkpoint. Verify that the original result is finalized after provider termination, with no new provider turn or repeated external action. diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 51e0c3af818..ad754e8e84e 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -54,9 +54,11 @@ import { type NativeExecutionInput, type PrpEvent, } from "@paperclipai/paperclip-runner"; +import { existsSync } from "node:fs"; import { createHash, generateKeyPairSync, randomUUID } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; import { nativeSha256 } from "./canonical.js"; +import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; import { RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError } from "./remote-runner-recovery-diagnostics.js"; import { agentDirectoryWorkingCopyService } from "../agent-directory-working-copies.js"; import { probeAgentDirectory } from "../agent-directory-probe.js"; @@ -1432,7 +1434,7 @@ describe("remote provider pack manifest", () => { productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", - }).map(([key, path]) => [key, { path, sha256: digest }])); + }).map(([key, path]) => [key, { path, sha256: digest }])) as Record<"nodeCommand" | "opencodeCommand", { path: string; sha256: string }>; const payload = { target: { platform: "linux", architecture }, runnerSourceRevision: "a".repeat(40), pins: { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion, @@ -1484,6 +1486,43 @@ describe("remote provider pack manifest", () => { sourcePath: state.bundledRunnerBinary, targetPath: "/workspace/.paperclip-runtime/paperclip-runner/bin/paperclip-runnerd", })] })]); expect(execute.mock.calls.some(([call]) => call.args?.[1] === "uname -s; uname -m")).toBe(false); + expect(existsSync(join(transport.stateDirectory!, "provider-pack-authority.json"))).toBe(false); + if (provider === "claude-x64") { + const identity = transport.prpIdentity!; + const artifact = { version: "0.0.0", digest: "sha256:" + createHash("sha256").update(daemon).digest("hex") }; + resolveRemoteProviderPackAuthority({ stateDirectory: transport.stateDirectory!, + binding: { companyId: remoteExecution.binding.companyId, agentId: remoteExecution.binding.agentId, + workspaceId: remoteExecution.binding.executionWorkspaceId, sessionId: remoteExecution.session.normalizedSessionId!, + runnerInstanceId: identity.runnerInstanceId, environmentLeaseId: identity.environmentLeaseId, + remoteCwd: "/workspace", providerDigest: nativeSha256(remoteExecution.provider) }, + reattach: false, current: () => manifest, runnerArtifact: () => artifact }).verified(); + await mkdir(join(transport.stateDirectory!, "control-plane"), { recursive: true }); + await writeFile(join(transport.stateDirectory!, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + const next = structuredClone(manifest); + next.payload.runnerSourceRevision = "b".repeat(40); + next.digest = "sha256:" + createHash("sha256").update(canonical(next.payload)).digest("hex"); + await writeFile(state.bundledPackManifestPath!, JSON.stringify(next)); + await writeFile(state.bundledRunnerBinary!, Buffer.concat([daemon, Buffer.from("new release")])); + const recoveryExecute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1]; + const stdout = script.includes("base64") ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "running" })).toString("base64") + : script.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : ""; + return { stdout, stderr: "", exitCode: 0, timedOut: false }; + }); + await createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, + runnerInstanceId: identity.runnerInstanceId, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "environment", leaseId: "lease", providerKey: "daytona", runner: { execute: recoveryExecute }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, + restartRecovery: { kind: "reattach_remote_runner", runId: remoteExecution.binding.runId, + leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "graceful", recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }, + }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ + providerPackAuthorityDigest: manifest.digest, runnerArtifact: artifact, adoptExistingRunner: { pid: 123 }, + }); + } } finally { state.bundledPackManifestPath = null; state.bundledRunnerBinary = null; await rm(root, { recursive: true, force: true }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index fe6d4732e76..cc03fcbfa56 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,5 @@ +import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; +import { verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import { writeExecutionControl } from "../execution-control-notifications.js"; @@ -47,7 +49,7 @@ import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; -import { prepareVerifiedRemoteProviderPack, useVerifiedPreinstalledRemoteProviderPack, runRemoteProviderVerification, shouldUseBundledAcpxImageAssets } from "./remote-provider-pack.js"; +import { prepareVerifiedRemoteProviderPack, useVerifiedPreinstalledRemoteProviderPack, shouldUseBundledAcpxImageAssets } from "./remote-provider-pack.js"; import { readProviderPackArchive, stageProviderPackArchive } from "./provider-pack-archive.js"; import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED, withReleasedNativeRunProcess } from "../native-local-process-stop.js"; @@ -12023,7 +12025,6 @@ async function createRunnerdBackendWithinSessionClaim( })); const configuredProviderPackRoot = input.runnerRemoteProviderPackPath?.trim() || remotePiCompanion?.providerPack || bundledClaudeRoot || null; - let expectedProviderPackManifest: RemoteProviderPackManifest | null = null; const useBundledRemoteImageAssets = (requiresRemoteProviderPack && !configuredProviderPackRoot && (input.execution.provider.kind === "opencode" || remoteClaude)) || @@ -12031,24 +12032,6 @@ async function createRunnerdBackendWithinSessionClaim( requiresRemoteProviderPack, configuredProviderPackRoot, provider: input.execution.provider, }); - if (useBundledRemoteImageAssets) { - expectedProviderPackManifest = bundledClaudeManifest && (remoteClaudeTarget === "linux-x64" || remoteClaudeTarget === "linux-arm64") - ? readBundledRemoteProviderPackManifest(bundledClaudeManifest, remoteClaudeTarget) - : readBundledRemoteProviderPackManifest(); - } else if (requiresRemoteProviderPack) { - if ( - !configuredProviderPackRoot || - !existsSync(configuredProviderPackRoot) || - !lstatSync(configuredProviderPackRoot).isDirectory() - ) { - throw new Error( - "runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH with the build-owned provider pack", - ); - } - expectedProviderPackManifest = readRemoteProviderPackManifest( - configuredProviderPackRoot, - ); - } const stagedRemoteProviderPackRoot = remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "provider-pack") : null; @@ -12056,6 +12039,25 @@ async function createRunnerdBackendWithinSessionClaim( const remoteBinary = remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "bin", "paperclip-runnerd") : null; + let currentProviderPackManifest: RemoteProviderPackManifest | undefined; + const resolveCurrentProviderPackManifest = () => { + if (currentProviderPackManifest) return currentProviderPackManifest; + currentProviderPackManifest = (() => { + if (useBundledRemoteImageAssets) { + return bundledClaudeManifest && (remoteClaudeTarget === "linux-x64" || remoteClaudeTarget === "linux-arm64") + ? readBundledRemoteProviderPackManifest(bundledClaudeManifest, remoteClaudeTarget) + : readBundledRemoteProviderPackManifest(); + } + if (!configuredProviderPackRoot || !existsSync(configuredProviderPackRoot) + || !lstatSync(configuredProviderPackRoot).isDirectory()) { + throw new Error("runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH with the build-owned provider pack"); + } + return readRemoteProviderPackManifest(configuredProviderPackRoot); + })(); + return currentProviderPackManifest; + }; + const reattachProviderPack = input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery !== undefined; + if (requiresRemoteProviderPack && !reattachProviderPack) resolveCurrentProviderPackManifest(); // The transport hashes runnerBinary on the controller before an external // launcher starts runnerd. Keep that artifact identity in the controller's // filesystem; the remote launcher separately owns the sandbox command path. @@ -12067,6 +12069,19 @@ async function createRunnerdBackendWithinSessionClaim( : remoteTarget ? remoteClaudeRunnerBinary || input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? bundledRemoteRunnerBinary() : resolvePaperclipRunnerBinary()) : resolvePaperclipRunnerBinary(); + const selectProviderPackAuthority = (reattach: boolean) => requiresRemoteProviderPack ? resolveRemoteProviderPackAuthority({ + stateDirectory: root, + binding: { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, + workspaceId: providerPackWorkspaceId(input.execution.binding.executionWorkspaceId, input.execution.binding.runId, effectiveEnvironmentLeaseId), sessionId: nativeSessionKey(input.execution), + runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, + remoteCwd: remoteTarget!.remoteCwd, providerDigest: nativeSha256(input.execution.provider) }, + reattach, + runnerArtifact: () => readRunnerdArtifactBinding(controllerRunnerBinary), + current: resolveCurrentProviderPackManifest, + }) : null; + let providerPackAuthority = selectProviderPackAuthority(reattachProviderPack); + let expectedProviderPackManifest = providerPackAuthority?.manifest ?? null; + let providerPackVerified = false; const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null; if (explicitRemoteCodex && remoteCodexNpmSpec) { @@ -12159,62 +12174,9 @@ async function createRunnerdBackendWithinSessionClaim( const verifyRemoteProviderPack = async (packRoot: string) => { if (!remoteTarget || !remoteCommandRunner || !expectedProviderPackManifest) return; - const expected = Buffer.from( - canonicalJson(expectedProviderPackManifest), - "utf8", - ).toString("base64"); - const providerNodeCommand = posix.join( - packRoot, - expectedProviderPackManifest.payload.artifacts.nodeCommand.path, - ); - const verifyScript = [ - "const fs=require('node:fs')", - "const crypto=require('node:crypto')", - "const path=require('node:path')", - "const root=process.argv[1]", - "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')", - "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()", - "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)", - "const manifest=JSON.parse(actual)", - "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')", - "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')", - "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", - "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}", - "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')", - "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}", - "const version=process.versions.node.split('.').map(Number)", - "const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)", - "if(version[0]JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version", - "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}", - "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')", - ].join(";"); - await runRemoteProviderVerification(() => remoteCommandRunner.execute({ - command: providerNodeCommand, - args: ["-e", verifyScript, packRoot, expected], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 30_000, - }), "provider pack verification"); - const opencodeCommand = posix.join( - packRoot, - expectedProviderPackManifest.payload.artifacts.opencodeCommand.path, - ); - const opencodeVersion = await runRemoteProviderVerification(() => remoteCommandRunner.execute({ - command: opencodeCommand, - args: ["--version"], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 30_000, - }), "OpenCode version verification"); - if ( - opencodeVersion.stdout.trim() !== REMOTE_PROVIDER_PACK_PINS.opencode - ) { - throw new Error( - "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", - ); - } + await verifyRemoteProviderPackIdentity({ runner: remoteCommandRunner, + cwd: remoteTarget.remoteCwd, packRoot, manifest: expectedProviderPackManifest }); + providerPackVerified = true; }; const discoverPreinstalledProviderPack = async () => { @@ -12333,7 +12295,7 @@ async function createRunnerdBackendWithinSessionClaim( ); } const archive = await measureNativeRunnerSpan(input.trace, "provider_pack.archive.verify", () => - readProviderPackArchive(configuredProviderPackRoot, expectedProviderPackManifest.digest), + readProviderPackArchive(configuredProviderPackRoot, expectedProviderPackManifest!.digest), ); const escapedPackRoot = stagedRemoteProviderPackRoot.replaceAll( "'", @@ -13243,6 +13205,7 @@ async function createRunnerdBackendWithinSessionClaim( // makes the fresh replacement look like ambiguous lost harness state. // Keep their evidence inside the same continuity-break archive. "failover-backups", + "provider-pack-authority.json", ]) { const source = resolve(root, name); if (existsSync(source)) renameSync(source, resolve(archiveRoot, name)); @@ -13265,6 +13228,9 @@ async function createRunnerdBackendWithinSessionClaim( } } remotePrepared = false; + providerPackAuthority = selectProviderPackAuthority(false); + expectedProviderPackManifest = providerPackAuthority?.manifest ?? null; + providerPackVerified = false; }; const localRecoveryProcess = input.restartRecovery?.kind === "reattach_existing_runner" @@ -13455,6 +13421,7 @@ async function createRunnerdBackendWithinSessionClaim( : {}), stateDirectory: root, runnerBinary: controllerRunnerBinary, + ...(providerPackAuthority ? { runnerArtifact: providerPackAuthority.runnerArtifact } : {}), codexCommand: remoteCodexBinary ?? undefined, sourceCodexHome: remoteTarget ? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, runnerEnvironment)) @@ -13660,12 +13627,22 @@ async function createRunnerdBackendWithinSessionClaim( priorAuthorityEpoch.revoke(); } sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch); + const recordAdmittedProviderPack = async () => { + if (providerPackVerified && providerPackAuthority) { + await recordAdmittedRemoteProviderPack(providerPackAuthority, message => input.onLog?.("stderr", message)); + } + }; return { bindManagedSession: wrapManagedSession, descriptor: () => backend.descriptor(), - openSession: async (sessionInput) => wrapManagedSession(await backend.openSession(sessionInput)), + openSession: async (sessionInput) => { + const session = await backend.openSession(sessionInput); + await recordAdmittedProviderPack(); + return wrapManagedSession(session); + }, recoverSession: async (snapshot, options) => { const result = backend.recoverSession ? await backend.recoverSession(snapshot, options) : { recovered: false, reason: "driver does not support recovery" }; + if (result.session) await recordAdmittedProviderPack(); return result.session ? { ...result, session: wrapManagedSession(result.session) } : result; }, openReplacementSession: async (sessionInput) => { @@ -13675,7 +13652,9 @@ async function createRunnerdBackendWithinSessionClaim( archiveContinuityState, { parentName: "native.session.execute" }, ); - return wrapManagedSession(await backend.openSession(sessionInput)); + const session = await backend.openSession(sessionInput); + await recordAdmittedProviderPack(); + return wrapManagedSession(session); }, } satisfies NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession }; } diff --git a/server/src/services/native-runtime/remote-provider-pack-authority.test.ts b/server/src/services/native-runtime/remote-provider-pack-authority.test.ts new file mode 100644 index 00000000000..f5f2039c4cf --- /dev/null +++ b/server/src/services/native-runtime/remote-provider-pack-authority.test.ts @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; +import { mkdtempSync, mkdirSync, rmSync, readFileSync, writeFileSync, symlinkSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; +import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; + +const directories: string[] = []; +afterEach(() => { for (const dir of directories.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +const binding = { companyId: "company", agentId: "agent", workspaceId: "workspace", sessionId: "session", + runnerInstanceId: "runner", environmentLeaseId: "lease", remoteCwd: "/workspace", providerDigest: "provider" }; +const runnerArtifact = () => ({ version: "0.0.0", digest: "sha256:" + "c".repeat(64) }); +const manifest = (revision: string) => { + const payload = { runnerSourceRevision: revision.repeat(40), target: { platform: "linux", architecture: "x64" }, + pins: { opencode: "1.18.34" }, artifacts: { nodeCommand: { path: "node", sha256: "sha256:" + "1".repeat(64) }, + opencodeCommand: { path: "opencode", sha256: "sha256:" + "2".repeat(64) } } }; + return { schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: "sha256:" + createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex") }; +}; +it("reattaches with the verified launch authority after the controller release changes", () => { + const stateDirectory = mkdtempSync(join(tmpdir(), "provider-pack-authority-")); directories.push(stateDirectory); + const original = manifest("a"), next = manifest("b"); + const launched = resolveRemoteProviderPackAuthority({ stateDirectory, binding, reattach: false, runnerArtifact, current: () => original }); + launched.verified(); + const current = vi.fn(() => next); + const recovered = resolveRemoteProviderPackAuthority({ stateDirectory, binding, reattach: true, current, runnerArtifact }); + expect(recovered.manifest).toEqual(original); + expect(recovered.runnerArtifact).toEqual(runnerArtifact()); + expect(current).not.toHaveBeenCalled(); +}); + +function fixture() { + const stateDirectory = mkdtempSync(join(tmpdir(), "provider-pack-authority-")); directories.push(stateDirectory); + const current = vi.fn(() => manifest("a")); + const input = { stateDirectory, binding, reattach: false, current, runnerArtifact }; + return { input, path: join(stateDirectory, "provider-pack-authority.json") }; +} +it("uses the new release for a fresh launch without changing the old receipt until verification", () => { + const { input, path } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + input.current.mockReturnValue(manifest("b")); + const fresh = resolveRemoteProviderPackAuthority(input); + expect(fresh.manifest).toEqual(manifest("b")); + expect(JSON.parse(readFileSync(path, "utf8")).manifest).toEqual(manifest("a")); + fresh.verified(); + expect(JSON.parse(readFileSync(path, "utf8")).manifest).toEqual(manifest("b")); + expect(statSync(path).mode & 0o777).toBe(0o600); +}); +it("keeps historical sessions without a receipt at the current release integrity gate", () => { + const { input } = fixture(); + const recovered = resolveRemoteProviderPackAuthority({ ...input, reattach: true }); + expect(recovered.manifest).toEqual(manifest("a")); + expect(input.current).toHaveBeenCalledOnce(); +}); +it.each(Object.keys(binding) as (keyof typeof binding)[])("rejects a saved authority for a different %s", field => { + const { input } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + expect(() => resolveRemoteProviderPackAuthority({ ...input, reattach: true, + binding: { ...binding, [field]: "other" } })).toThrow("does not match this execution"); + expect(input.current).toHaveBeenCalledOnce(); +}); +it.each(["digest", "schema", "symlink", "oversized", "invalid_json"])("fails closed on %s instead of trusting remote or current bytes", kind => { + const { input, path } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + const saved = JSON.parse(readFileSync(path, "utf8")); + if (kind === "digest") saved.manifest.payload.runnerSourceRevision = "c".repeat(40); + if (kind === "schema") saved.schema = "unknown"; + writeFileSync(path, JSON.stringify(saved)); + if (kind === "invalid_json") writeFileSync(path, "{"); + if (kind === "oversized") writeFileSync(path, " ".repeat(512 * 1024 + 1)); + if (kind === "symlink") { const copy = path + ".copy"; writeFileSync(copy, JSON.stringify(saved)); rmSync(path); symlinkSync(copy, path); } + expect(() => resolveRemoteProviderPackAuthority({ ...input, reattach: true })).toThrow("saved provider pack authority"); + expect(input.current).toHaveBeenCalledOnce(); +}); + +it("uses current authority for a replacement lease instead of restoring the prior pack", () => { + const { input, path } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + const currentRunner = { version: "0.0.1", digest: "sha256:" + "d".repeat(64) }; + const replacement = resolveRemoteProviderPackAuthority({ ...input, binding: { ...binding, environmentLeaseId: "replacement" }, + current: () => manifest("b"), runnerArtifact: () => currentRunner }); + expect(replacement.manifest).toEqual(manifest("b")); + expect(replacement.runnerArtifact).toEqual(currentRunner); + replacement.verified(); + expect(JSON.parse(readFileSync(path, "utf8")).binding.environmentLeaseId).toBe("replacement"); +}); +it("retains the authenticated daemon identity across controller daemon replacement", () => { + const { input } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + const changedRunner = vi.fn(() => ({ version: "new", digest: "sha256:" + "d".repeat(64) })); + const recovered = resolveRemoteProviderPackAuthority({ ...input, reattach: true, runnerArtifact: changedRunner }); + expect(recovered.runnerArtifact).toEqual(runnerArtifact()); + expect(changedRunner).not.toHaveBeenCalled(); +}); +it.each([{}, { version: "0.0.0", digest: "not-a-sha256" }])("rejects an invalid saved daemon identity: %j", runnerArtifact => { + const { input, path } = fixture(); + resolveRemoteProviderPackAuthority(input).verified(); + const saved = JSON.parse(readFileSync(path, "utf8")); + writeFileSync(path, JSON.stringify({ ...saved, runnerArtifact })); + expect(() => resolveRemoteProviderPackAuthority({ ...input, reattach: true })).toThrow("saved provider pack authority"); +}); + +it("keeps admitted session ownership with the caller if receipt storage or its warning fails", async () => { + const { input, path } = fixture(); + mkdirSync(path); + const authority = resolveRemoteProviderPackAuthority(input); + const warning = vi.fn(() => { throw new Error("log unavailable"); }); + await expect(recordAdmittedRemoteProviderPack(authority, warning)).resolves.toBe(false); + expect(warning).toHaveBeenCalledWith(expect.stringContaining("could not save remote release authority")); + expect(() => resolveRemoteProviderPackAuthority({ ...input, reattach: true })).toThrow("saved provider pack authority"); +}); +it("retains the original projectless lease across warm run placeholders but rejects a different real workspace", () => { + const { input } = fixture(); + const first = { ...binding, workspaceId: providerPackWorkspaceId("run-1", "run-1", "original-lease") }; + resolveRemoteProviderPackAuthority({ ...input, binding: first }).verified(); + const next = { ...binding, workspaceId: providerPackWorkspaceId("run-2", "run-2", "original-lease") }; + expect(resolveRemoteProviderPackAuthority({ ...input, binding: next, reattach: true }).manifest).toEqual(manifest("a")); + const changed = { ...binding, workspaceId: providerPackWorkspaceId("project-workspace", "run-2", "original-lease") }; + expect(() => resolveRemoteProviderPackAuthority({ ...input, binding: changed, reattach: true })).toThrow("does not match this execution"); +}); diff --git a/server/src/services/native-runtime/remote-provider-pack-authority.ts b/server/src/services/native-runtime/remote-provider-pack-authority.ts new file mode 100644 index 00000000000..5516ad6d959 --- /dev/null +++ b/server/src/services/native-runtime/remote-provider-pack-authority.ts @@ -0,0 +1,107 @@ +import { createHash, randomUUID } from "node:crypto"; +import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { redactSensitiveText } from "../../redaction.js"; +import { canonicalProviderPackJson, type ProviderPackIdentity } from "./remote-provider-pack-verification.js"; + +export type ProviderPackBinding = { + companyId: string; + agentId: string; + workspaceId: string; + sessionId: string; + runnerInstanceId: string; + environmentLeaseId: string; + remoteCwd: string; + providerDigest: string; +}; +export type RunnerArtifactIdentity = { version: string; digest: string }; + +const schema = "paperclip.remote-provider-pack-authority.v1"; +const filename = "provider-pack-authority.json"; +const maximumBytes = 512 * 1024; + +function assertManifest(manifest: ProviderPackIdentity): void { + if (manifest?.schema !== "paperclip-runner/remote-provider-pack/v1" || !manifest.payload + || manifest.digest !== `sha256:${createHash("sha256").update(canonicalProviderPackJson(manifest.payload)).digest("hex")}`) { + throw new Error("runner_remote_provider_artifact_incompatible: saved provider pack authority is invalid"); + } +} + +export function resolveRemoteProviderPackAuthority(input: { + stateDirectory: string; + binding: ProviderPackBinding; + reattach: boolean; + current: () => T; + runnerArtifact: () => RunnerArtifactIdentity; +}): { manifest: T; runnerArtifact: RunnerArtifactIdentity; verified: () => void } { + const root = lstatSync(input.stateDirectory); + if (!root.isDirectory() || root.isSymbolicLink()) { + throw new Error("runner_remote_provider_artifact_incompatible: provider pack authority directory is unsafe"); + } + const path = join(input.stateDirectory, filename); + let manifest: T | undefined; + let runnerArtifact: RunnerArtifactIdentity | undefined; + if (input.reattach) { + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > maximumBytes) throw new Error("invalid authority file"); + const saved = JSON.parse(readFileSync(fd, "utf8")); + if (saved.schema !== schema || canonicalProviderPackJson(saved.binding) !== canonicalProviderPackJson(input.binding)) { + throw new Error("authority binding mismatch"); + } + assertManifest(saved.manifest); + if (typeof saved.runnerArtifact?.version !== "string" || !saved.runnerArtifact.version + || !/^sha256:[a-f0-9]{64}$/.test(saved.runnerArtifact.digest)) throw new Error("invalid runner artifact identity"); + manifest = saved.manifest as T; + runnerArtifact = saved.runnerArtifact; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw new Error("runner_remote_provider_artifact_incompatible: saved provider pack authority is invalid or does not match this execution", { cause: error }); + } + } finally { + if (fd !== undefined) closeSync(fd); + } + } + manifest ??= input.current(); + assertManifest(manifest); + runnerArtifact ??= input.runnerArtifact(); + const selectedRunner = structuredClone(runnerArtifact); + const selected = structuredClone(manifest); + const receipt = JSON.stringify({ schema, binding: input.binding, manifest: selected, runnerArtifact: selectedRunner }); + if (Buffer.byteLength(receipt) > maximumBytes) throw new Error("runner_remote_provider_artifact_incompatible: provider pack authority exceeds its size limit"); + return { + manifest: selected, + runnerArtifact: selectedRunner, + verified: () => { + const temporary = `${path}.${randomUUID()}.tmp`; + try { + writeFileSync(temporary, receipt, { flag: "wx", mode: 0o600 }); + renameSync(temporary, path); + } finally { + rmSync(temporary, { force: true }); + } + }, + }; +} + +export function providerPackWorkspaceId(workspaceId: string, runId: string, environmentLeaseId: string): string { + return workspaceId === runId ? environmentLeaseId : workspaceId; +} + +export async function recordAdmittedRemoteProviderPack( + authority: { verified: () => void }, + onWarning: (message: string) => void | Promise, +): Promise { + try { + authority.verified(); + return true; + } catch (error) { + const reason = redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(-1_024); + try { + await onWarning(`[paperclip-runner] could not save remote release authority; recovery still requires a verified receipt or current release artifacts: ${reason}\n`); + } catch {} + return false; + } +} diff --git a/server/src/services/native-runtime/remote-provider-pack-verification.ts b/server/src/services/native-runtime/remote-provider-pack-verification.ts new file mode 100644 index 00000000000..3f504a636e9 --- /dev/null +++ b/server/src/services/native-runtime/remote-provider-pack-verification.ts @@ -0,0 +1,85 @@ +import { posix } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { runRemoteProviderVerification } from "./remote-provider-pack.js"; + +export function canonicalProviderPackJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalProviderPackJson).join(",")}]`; + if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalProviderPackJson(entry)}`).join(",")}}`; + return JSON.stringify(value) ?? "null"; +} + +export type ProviderPackIdentity = { + schema: string; + digest: string; + payload: { + pins: { opencode: string; [key: string]: string | null }; + target: { platform: string; architecture: string }; + runnerSourceRevision: string; + artifacts: { nodeCommand: { path: string; sha256: string }; opencodeCommand: { path: string; sha256: string }; [key: string]: { path: string; sha256: string } }; + }; +}; + +export async function verifyRemoteProviderPack({ runner, cwd, packRoot, manifest }: { + runner: CommandManagedRuntimeRunner; + cwd: string; + packRoot: string; + manifest: ProviderPackIdentity; +}): Promise { + const expected = Buffer.from( + canonicalProviderPackJson(manifest), + "utf8", + ).toString("base64"); + const providerNodeCommand = posix.join( + packRoot, + manifest.payload.artifacts.nodeCommand.path, + ); + const verifyScript = [ + "const fs=require('node:fs')", + "const crypto=require('node:crypto')", + "const path=require('node:path')", + "const root=process.argv[1]", + "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')", + "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()", + "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)", + "const manifest=JSON.parse(actual)", + "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')", + "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')", + "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", + "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}", + "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')", + "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}", + "const version=process.versions.node.split('.').map(Number)", + "const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)", + "if(version[0]JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version", + "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}", + "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')", + ].join(";"); + await runRemoteProviderVerification(() => runner.execute({ + command: providerNodeCommand, + args: ["-e", verifyScript, packRoot, expected], + cwd: cwd, + bypassSession: true, + timeoutMs: 30_000, + }), "provider pack verification"); + const opencodeCommand = posix.join( + packRoot, + manifest.payload.artifacts.opencodeCommand.path, + ); + const opencodeVersion = await runRemoteProviderVerification(() => runner.execute({ + command: opencodeCommand, + args: ["--version"], + cwd: cwd, + bypassSession: true, + timeoutMs: 30_000, + }), "OpenCode version verification"); + if ( + opencodeVersion.stdout.trim() !== manifest.payload.pins.opencode + ) { + throw new Error( + "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", + ); + } +} diff --git a/server/src/services/native-runtime/remote-provider-pack.test.ts b/server/src/services/native-runtime/remote-provider-pack.test.ts index a4ec1f60d97..5d5991abca4 100644 --- a/server/src/services/native-runtime/remote-provider-pack.test.ts +++ b/server/src/services/native-runtime/remote-provider-pack.test.ts @@ -25,6 +25,14 @@ describe("remote provider verification", () => { await expect(runRemoteProviderVerification(execute, "pack")).rejects.toThrow("runner_remote_provider_artifact_incompatible: pack failed (exitCode=1"); expect(execute).toHaveBeenCalledOnce(); }); + it("retains a bounded redacted rejection from a combined-output transport", async () => { + const execute = vi.fn().mockResolvedValue({ ...ok, exitCode: 1, + stdout: "Error: manifest mismatch\nANTHROPIC_API_KEY=secret-fixture-value", stderr: "" }); + const failure = await runRemoteProviderVerification(execute, "pack").catch(error => error); + expect(failure.message).toContain("manifest mismatch"); + expect(failure.message).not.toContain("secret-fixture-value"); + expect(execute).toHaveBeenCalledOnce(); + }); it("recovers a transport exception and propagates cancellation", async () => { const execute = vi.fn().mockRejectedValueOnce(new Error("connection unavailable")).mockResolvedValueOnce(ok); await expect(runRemoteProviderVerification(execute, "pack")).resolves.toEqual(ok); diff --git a/server/src/services/native-runtime/remote-provider-pack.ts b/server/src/services/native-runtime/remote-provider-pack.ts index b7045ad42d7..f6621e1e1ea 100644 --- a/server/src/services/native-runtime/remote-provider-pack.ts +++ b/server/src/services/native-runtime/remote-provider-pack.ts @@ -1,3 +1,4 @@ +import { redactSensitiveText } from "../../redaction.js"; import type { NativeRunTrace } from "./native-run-trace.js"; import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; @@ -37,10 +38,10 @@ export async function runRemoteProviderVerification( result = await execute(); } catch (error) { if (error instanceof Error && error.name === "AbortError") throw error; - diagnostic = error instanceof Error ? error.message : String(error); + diagnostic = redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(-1_024); continue; } - diagnostic = `exitCode=${result.exitCode}; timedOut=${result.timedOut}; signal=${result.signal ?? "none"}; ${result.stderr.trim().slice(-1_024)}`; + diagnostic = `exitCode=${result.exitCode}; timedOut=${result.timedOut}; signal=${result.signal ?? "none"}; ${redactSensitiveText([result.stderr, result.stdout].filter(Boolean).join("\n")).trim().slice(-1_024)}`; if (result.timedOut || result.exitCode === null || result.signal) continue; if (result.exitCode !== 0) { throw new Error(`runner_remote_provider_artifact_incompatible: ${label} failed (${diagnostic})`);