From 8b7026152199dd45115487f6bd3021ef3115ed78 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 22:36:14 -0500 Subject: [PATCH] fix: distinguish provider cache mismatches from unavailable verification --- .../native-runtime/native-session-executor.ts | 33 ++++-------- .../remote-provider-pack-cache.test.ts | 50 ++++++++++++++++++- .../remote-provider-pack-cache.ts | 42 +++++++++++++++- 3 files changed, 100 insertions(+), 25 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0675e50c065..a22f0aa0abe 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -42,7 +42,7 @@ import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js"; -import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js"; +import { assertRemoteProviderPackVerificationResult, computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js"; import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js"; import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; @@ -12295,20 +12295,21 @@ async function createRunnerdBackendWithinSessionClaim( "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')", "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()", "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)", + "const fail=(reason)=>{fs.writeSync(2,'paperclip-provider-pack-verification:'+reason);process.exit(42)}", "const manifest=JSON.parse(actual)", - "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')", + "if(canonical(manifest)!==expected)fail('manifest_mismatch')", "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')", "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", - "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}", - "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')", - "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}", + "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)fail('artifact_digest_mismatch')}", + "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)fail('dist_digest_mismatch')", + "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)fail('candidate_digest_mismatch')}", "const version=process.versions.node.split('.').map(Number)", `const minimums=[manifest.payload.pins.nodeMinimum,${JSON.stringify(REMOTE_PROVIDER_PACK_PINS.nodeMinimum)}]`, - "for(const value of minimums){const minimum=value.split('.').map(Number);if(version[0]JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version", "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}", - "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')", + "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)fail('package_version_mismatch')", ].join(";"); const verified = await remoteCommandRunner.execute({ command: providerNodeCommand, @@ -12317,11 +12318,7 @@ async function createRunnerdBackendWithinSessionClaim( bypassSession: true, timeoutMs: 30_000, }); - if (verified.exitCode !== 0 || verified.timedOut) { - throw new Error( - `runner_remote_provider_artifact_incompatible: provider pack verification failed (${verified.stderr.trim().slice(-1_024)})`, - ); - } + assertRemoteProviderPackVerificationResult(verified); const opencodeCommand = posix.join( packRoot, expectedProviderPackManifest.payload.artifacts.opencodeCommand.path, @@ -12333,15 +12330,7 @@ async function createRunnerdBackendWithinSessionClaim( bypassSession: true, timeoutMs: 30_000, }); - if ( - opencodeVersion.exitCode !== 0 || - opencodeVersion.timedOut || - opencodeVersion.stdout.trim() !== expectedProviderPackManifest.payload.pins.opencode - ) { - throw new Error( - "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", - ); - } + assertRemoteProviderPackVerificationResult(opencodeVersion, expectedProviderPackManifest.payload.pins.opencode); }; const discoverPreinstalledProviderPack = async () => { diff --git a/server/src/services/native-runtime/remote-provider-pack-cache.test.ts b/server/src/services/native-runtime/remote-provider-pack-cache.test.ts index 9fad5275291..0665266083b 100644 --- a/server/src/services/native-runtime/remote-provider-pack-cache.test.ts +++ b/server/src/services/native-runtime/remote-provider-pack-cache.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; -import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js"; +import { RemoteProviderPackVerificationError, assertRemoteProviderPackVerificationResult, computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js"; const roots: string[] = []; const digest = `sha256:${"a".repeat(64)}`; @@ -103,7 +103,7 @@ async function fixture() { return { cacheRoot: computerProviderPackCachePath(home, digest), sessionKey: "task-a", owner: { ownerId: "owner-a", generation: 1 }, runner, - verify: async (root: string) => { expect(await fs.readFile(join(root, "payload"), "utf8")).toBe("verified bytes"); }, + verify: async (root: string) => { if (await fs.readFile(join(root, "payload"), "utf8") !== "verified bytes") throw new RemoteProviderPackVerificationError("mismatch", "artifact_digest_mismatch"); }, stage: async (root: string, guarded: CommandManagedRuntimeRunner) => { const result = await guarded.execute({ command: "python3", args: ["-c", "import os,sys;os.mkdir(sys.argv[1],0o700);open(os.path.join(sys.argv[1],'payload'),'w').write(sys.stdin.read())", root], stdin: "verified bytes" }); if (result.exitCode !== 0) throw new Error(result.stderr); @@ -140,6 +140,23 @@ describe("persistent computer provider pack cache", () => { expect(await fs.readFile(join(input.cacheRoot, "payload"), "utf8")).toBe("corrupt"); }); + it.each(["timeout", "transport", "command", "compatibility"])("preserves the shared pack without a deletion recommendation after %s failure", async (kind) => { + const input = await fixture(); + await fs.mkdir(input.cacheRoot, { recursive: true }); + await fs.writeFile(join(input.cacheRoot, "payload"), "verified bytes"); + const secretCause = new Error("private command/path/token"); + const failure = kind === "transport" ? secretCause + : new RemoteProviderPackVerificationError(kind === "compatibility" ? "incompatible" : "unavailable", + kind === "timeout" ? "timeout" : kind === "compatibility" ? "node_version_incompatible" : "command_failed", secretCause); + const stage = vi.fn(input.stage); + const error = await prepareComputerProviderPackCache({ ...input, stage, verify: async () => { throw failure; } }).catch(error => error); + expect(error).toBeInstanceOf(RemoteProviderPackVerificationError); + expect(error.message).not.toMatch(/corrupt|remove|quarantine|private command/); + expect(error.cause).toBe(kind === "transport" ? failure : secretCause); + expect(stage).not.toHaveBeenCalled(); + expect(await fs.readFile(join(input.cacheRoot, "payload"), "utf8")).toBe("verified bytes"); + }); + it("publishes without replacement when two tasks race and verifies the winner", async () => { const input = await fixture(); const both = deferred(); let staged = 0; const stage = async (root: string, guarded: CommandManagedRuntimeRunner) => { @@ -221,3 +238,32 @@ describe("persistent computer provider pack cache", () => { await expect(fs.stat(attempted)).rejects.toMatchObject({ code: "ENOENT" }); }); }); + +describe("remote provider verification result classification", () => { + const result = { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + it.each(["manifest_mismatch", "artifact_digest_mismatch", "dist_digest_mismatch", "candidate_digest_mismatch", "package_version_mismatch"])("recognizes only an explicit %s result", (reason) => { + expect(() => assertRemoteProviderPackVerificationResult({ ...result, exitCode: 42, stderr: `paperclip-provider-pack-verification:${reason}` })) + .toThrow(expect.objectContaining({ kind: "mismatch", reason })); + }); + it.each([ + { exitCode: 42, timedOut: true, stderr: "paperclip-provider-pack-verification:artifact_digest_mismatch" }, + { exitCode: 255, timedOut: false, stderr: "private SSH stderr" }, + { exitCode: 1, timedOut: false, stderr: "Error: artifact digest mismatch" }, + { exitCode: 42, timedOut: false, stderr: "paperclip-provider-pack-verification:unknown" }, + { exitCode: 42, timedOut: false, stderr: "paperclip-provider-pack-verification:artifact_digest_mismatch\nprivate stderr" }, + ])("does not classify incomplete or unexpected results as corruption: %j", (failure) => { + const error = (() => { try { assertRemoteProviderPackVerificationResult({ ...result, ...failure }); } catch (error) { return error; } })(); + expect(error).toMatchObject({ kind: "unavailable" }); + expect((error as Error).message).not.toMatch(/private|digest_mismatch|quarantine/); + }); + it("distinguishes a completed OpenCode version mismatch from a failed or timed-out version probe", () => { + expect(() => assertRemoteProviderPackVerificationResult({ ...result, stdout: "1.2.3" }, "1.2.3")).not.toThrow(); + expect(() => assertRemoteProviderPackVerificationResult({ ...result, stdout: "1.2.2" }, "1.2.3")).toThrow(expect.objectContaining({ kind: "mismatch", reason: "opencode_version_mismatch" })); + for (const failure of [{ exitCode: 1 }, { timedOut: true }]) + expect(() => assertRemoteProviderPackVerificationResult({ ...result, ...failure }, "1.2.3")).toThrow(expect.objectContaining({ kind: "unavailable" })); + }); + it.each(["node_version_incompatible", "target_mismatch"])("keeps %s distinct from cache corruption", reason => { + expect(() => assertRemoteProviderPackVerificationResult({ ...result, exitCode: 42, stderr: `paperclip-provider-pack-verification:${reason}` })) + .toThrow(expect.objectContaining({ kind: "incompatible", reason })); + }); +}); diff --git a/server/src/services/native-runtime/remote-provider-pack-cache.ts b/server/src/services/native-runtime/remote-provider-pack-cache.ts index 2fd696a305b..70d51432e8d 100644 --- a/server/src/services/native-runtime/remote-provider-pack-cache.ts +++ b/server/src/services/native-runtime/remote-provider-pack-cache.ts @@ -2,6 +2,41 @@ import { createHash, randomUUID } from "node:crypto"; import { posix } from "node:path"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +const providerPackMismatchReasons = ["manifest_mismatch", "artifact_digest_mismatch", "dist_digest_mismatch", "candidate_digest_mismatch", "package_version_mismatch", "opencode_version_mismatch"] as const; +const providerPackCompatibilityReasons = ["node_version_incompatible", "target_mismatch"] as const; +type ProviderPackVerificationReason = typeof providerPackMismatchReasons[number] | typeof providerPackCompatibilityReasons[number] | "timeout" | "command_failed" | "transport_error"; + +export class RemoteProviderPackVerificationError extends Error { + constructor(readonly kind: "mismatch" | "incompatible" | "unavailable", readonly reason: ProviderPackVerificationReason, cause?: unknown) { + super(`runner_remote_provider_${kind === "unavailable" ? "verification_unavailable" : "artifact_incompatible"}: ${reason}`, { cause }); + } +} + +/** Only an explicit bounded verifier result proves a mismatch. SSH failures, + * timeouts and unexpected stderr do not establish anything about cached bytes. */ +export function assertRemoteProviderPackVerificationResult( + result: Pick>, "exitCode" | "timedOut" | "stdout" | "stderr">, + expectedOpenCodeVersion?: string, +): void { + if (result.timedOut) throw new RemoteProviderPackVerificationError("unavailable", "timeout", result); + if (result.exitCode === 0) { + if (expectedOpenCodeVersion !== undefined && result.stdout.trim() !== expectedOpenCodeVersion) + throw new RemoteProviderPackVerificationError("mismatch", "opencode_version_mismatch", result); + return; + } + if (result.exitCode === 42 && expectedOpenCodeVersion === undefined) { + for (const reason of providerPackMismatchReasons) { + if (result.stderr.trim() === `paperclip-provider-pack-verification:${reason}`) + throw new RemoteProviderPackVerificationError("mismatch", reason, result); + } + for (const reason of providerPackCompatibilityReasons) { + if (result.stderr.trim() === `paperclip-provider-pack-verification:${reason}`) + throw new RemoteProviderPackVerificationError("incompatible", reason, result); + } + } + throw new RemoteProviderPackVerificationError("unavailable", "command_failed", result); +} + /** The controller's retained attachment template pins content-addressed paths. * Remote manifests may supply bytes, but cannot choose a new code authority. */ export function pinnedComputerProviderPack(input: { @@ -232,7 +267,12 @@ export async function prepareComputerProviderPackCache(input: { }; const verifyShared = async () => { try { await input.verify(input.cacheRoot); } - catch (cause) { throw new Error(`runner_remote_provider_cache_corrupt: ${input.cacheRoot}; stop tasks using this pack and have an operator remove or quarantine this exact cache directory before retrying`, { cause }); } + catch (cause) { + if (cause instanceof RemoteProviderPackVerificationError && cause.kind === "mismatch") + throw new Error(`runner_remote_provider_cache_corrupt: ${cause.reason}; ${input.cacheRoot}; stop tasks using this pack and have an operator remove or quarantine this exact cache directory before retrying`, { cause }); + if (cause instanceof RemoteProviderPackVerificationError) throw cause; + throw new RemoteProviderPackVerificationError("unavailable", "transport_error", cause); + } }; if (await operation("exists") === "true") { await verifyShared();