fix: distinguish provider cache mismatches from unavailable verification

server/src/services/native-runtime/native-session-executor.ts

@@ -42,7 +42,7 @@ import { nativeCompletionFeedback } from "./native-completion-feedback.js";
 import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
 import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js";
 import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js";
-import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js";
+import { assertRemoteProviderPackVerificationResult, computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js";
 import { selectRemotePiCompanion } from "./remote-pi-companion.js";
 import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js";
 import { remoteLeaseCleanupScope } from "../remote-execution-termination.js";
@@ -12295,20 +12295,21 @@ async function createRunnerdBackendWithinSessionClaim(
       "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')",
       "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
       "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
+      "const fail=(reason)=>{fs.writeSync(2,'paperclip-provider-pack-verification:'+reason);process.exit(42)}",
       "const manifest=JSON.parse(actual)",
-      "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')",
+      "if(canonical(manifest)!==expected)fail('manifest_mismatch')",
       "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
       "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
-      "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
-      "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
-      "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}",
+      "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)fail('artifact_digest_mismatch')}",
+      "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)fail('dist_digest_mismatch')",
+      "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)fail('candidate_digest_mismatch')}",
       "const version=process.versions.node.split('.').map(Number)",
       `const minimums=[manifest.payload.pins.nodeMinimum,${JSON.stringify(REMOTE_PROVIDER_PACK_PINS.nodeMinimum)}]`,
-      "for(const value of minimums){const minimum=value.split('.').map(Number);if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')}",
-      "if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
+      "for(const value of minimums){const minimum=value.split('.').map(Number);if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))fail('node_version_incompatible')}",
+      "if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)fail('target_mismatch')",
       "const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
       "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
-      "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
+      "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)fail('package_version_mismatch')",
     ].join(";");
     const verified = await remoteCommandRunner.execute({
       command: providerNodeCommand,
@@ -12317,11 +12318,7 @@ async function createRunnerdBackendWithinSessionClaim(
       bypassSession: true,
       timeoutMs: 30_000,
     });
-    if (verified.exitCode !== 0 || verified.timedOut) {
-      throw new Error(
-        `runner_remote_provider_artifact_incompatible: provider pack verification failed (${verified.stderr.trim().slice(-1_024)})`,
-      );
-    }
+    assertRemoteProviderPackVerificationResult(verified);
     const opencodeCommand = posix.join(
       packRoot,
       expectedProviderPackManifest.payload.artifacts.opencodeCommand.path,
@@ -12333,15 +12330,7 @@ async function createRunnerdBackendWithinSessionClaim(
       bypassSession: true,
       timeoutMs: 30_000,
     });
-    if (
-      opencodeVersion.exitCode !== 0 ||
-      opencodeVersion.timedOut ||
-      opencodeVersion.stdout.trim() !== expectedProviderPackManifest.payload.pins.opencode
-    ) {
-      throw new Error(
-        "runner_remote_provider_artifact_incompatible: OpenCode version mismatch",
-      );
-    }
+    assertRemoteProviderPackVerificationResult(opencodeVersion, expectedProviderPackManifest.payload.pins.opencode);
   };
 
   const discoverPreinstalledProviderPack = async () => {

server/src/services/native-runtime/remote-provider-pack-cache.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
-import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js";
+import { RemoteProviderPackVerificationError, assertRemoteProviderPackVerificationResult, computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js";
 
 const roots: string[] = [];
 const digest = `sha256:${"a".repeat(64)}`;
@@ -103,7 +103,7 @@ async function fixture() {
   return {
     cacheRoot: computerProviderPackCachePath(home, digest), sessionKey: "task-a",
     owner: { ownerId: "owner-a", generation: 1 }, runner,
-    verify: async (root: string) => { expect(await fs.readFile(join(root, "payload"), "utf8")).toBe("verified bytes"); },
+    verify: async (root: string) => { if (await fs.readFile(join(root, "payload"), "utf8") !== "verified bytes") throw new RemoteProviderPackVerificationError("mismatch", "artifact_digest_mismatch"); },
     stage: async (root: string, guarded: CommandManagedRuntimeRunner) => {
       const result = await guarded.execute({ command: "python3", args: ["-c", "import os,sys;os.mkdir(sys.argv[1],0o700);open(os.path.join(sys.argv[1],'payload'),'w').write(sys.stdin.read())", root], stdin: "verified bytes" });
       if (result.exitCode !== 0) throw new Error(result.stderr);
@@ -140,6 +140,23 @@ describe("persistent computer provider pack cache", () => {
     expect(await fs.readFile(join(input.cacheRoot, "payload"), "utf8")).toBe("corrupt");
   });
 
+  it.each(["timeout", "transport", "command", "compatibility"])("preserves the shared pack without a deletion recommendation after %s failure", async (kind) => {
+    const input = await fixture();
+    await fs.mkdir(input.cacheRoot, { recursive: true });
+    await fs.writeFile(join(input.cacheRoot, "payload"), "verified bytes");
+    const secretCause = new Error("private command/path/token");
+    const failure = kind === "transport" ? secretCause
+      : new RemoteProviderPackVerificationError(kind === "compatibility" ? "incompatible" : "unavailable",
+        kind === "timeout" ? "timeout" : kind === "compatibility" ? "node_version_incompatible" : "command_failed", secretCause);
+    const stage = vi.fn(input.stage);
+    const error = await prepareComputerProviderPackCache({ ...input, stage, verify: async () => { throw failure; } }).catch(error => error);
+    expect(error).toBeInstanceOf(RemoteProviderPackVerificationError);
+    expect(error.message).not.toMatch(/corrupt|remove|quarantine|private command/);
+    expect(error.cause).toBe(kind === "transport" ? failure : secretCause);
+    expect(stage).not.toHaveBeenCalled();
+    expect(await fs.readFile(join(input.cacheRoot, "payload"), "utf8")).toBe("verified bytes");
+  });
+
   it("publishes without replacement when two tasks race and verifies the winner", async () => {
     const input = await fixture(); const both = deferred(); let staged = 0;
     const stage = async (root: string, guarded: CommandManagedRuntimeRunner) => {
@@ -221,3 +238,32 @@ describe("persistent computer provider pack cache", () => {
     await expect(fs.stat(attempted)).rejects.toMatchObject({ code: "ENOENT" });
   });
 });
+
+describe("remote provider verification result classification", () => {
+  const result = { exitCode: 0, timedOut: false, stdout: "", stderr: "" };
+  it.each(["manifest_mismatch", "artifact_digest_mismatch", "dist_digest_mismatch", "candidate_digest_mismatch", "package_version_mismatch"])("recognizes only an explicit %s result", (reason) => {
+    expect(() => assertRemoteProviderPackVerificationResult({ ...result, exitCode: 42, stderr: `paperclip-provider-pack-verification:${reason}` }))
+      .toThrow(expect.objectContaining({ kind: "mismatch", reason }));
+  });
+  it.each([
+    { exitCode: 42, timedOut: true, stderr: "paperclip-provider-pack-verification:artifact_digest_mismatch" },
+    { exitCode: 255, timedOut: false, stderr: "private SSH stderr" },
+    { exitCode: 1, timedOut: false, stderr: "Error: artifact digest mismatch" },
+    { exitCode: 42, timedOut: false, stderr: "paperclip-provider-pack-verification:unknown" },
+    { exitCode: 42, timedOut: false, stderr: "paperclip-provider-pack-verification:artifact_digest_mismatch\nprivate stderr" },
+  ])("does not classify incomplete or unexpected results as corruption: %j", (failure) => {
+    const error = (() => { try { assertRemoteProviderPackVerificationResult({ ...result, ...failure }); } catch (error) { return error; } })();
+    expect(error).toMatchObject({ kind: "unavailable" });
+    expect((error as Error).message).not.toMatch(/private|digest_mismatch|quarantine/);
+  });
+  it("distinguishes a completed OpenCode version mismatch from a failed or timed-out version probe", () => {
+    expect(() => assertRemoteProviderPackVerificationResult({ ...result, stdout: "1.2.3" }, "1.2.3")).not.toThrow();
+    expect(() => assertRemoteProviderPackVerificationResult({ ...result, stdout: "1.2.2" }, "1.2.3")).toThrow(expect.objectContaining({ kind: "mismatch", reason: "opencode_version_mismatch" }));
+    for (const failure of [{ exitCode: 1 }, { timedOut: true }])
+      expect(() => assertRemoteProviderPackVerificationResult({ ...result, ...failure }, "1.2.3")).toThrow(expect.objectContaining({ kind: "unavailable" }));
+  });
+  it.each(["node_version_incompatible", "target_mismatch"])("keeps %s distinct from cache corruption", reason => {
+    expect(() => assertRemoteProviderPackVerificationResult({ ...result, exitCode: 42, stderr: `paperclip-provider-pack-verification:${reason}` }))
+      .toThrow(expect.objectContaining({ kind: "incompatible", reason }));
+  });
+});

server/src/services/native-runtime/remote-provider-pack-cache.ts

@@ -2,6 +2,41 @@ import { createHash, randomUUID } from "node:crypto";
 import { posix } from "node:path";
 import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
 
+const providerPackMismatchReasons = ["manifest_mismatch", "artifact_digest_mismatch", "dist_digest_mismatch", "candidate_digest_mismatch", "package_version_mismatch", "opencode_version_mismatch"] as const;
+const providerPackCompatibilityReasons = ["node_version_incompatible", "target_mismatch"] as const;
+type ProviderPackVerificationReason = typeof providerPackMismatchReasons[number] | typeof providerPackCompatibilityReasons[number] | "timeout" | "command_failed" | "transport_error";
+
+export class RemoteProviderPackVerificationError extends Error {
+  constructor(readonly kind: "mismatch" | "incompatible" | "unavailable", readonly reason: ProviderPackVerificationReason, cause?: unknown) {
+    super(`runner_remote_provider_${kind === "unavailable" ? "verification_unavailable" : "artifact_incompatible"}: ${reason}`, { cause });
+  }
+}
+
+/** Only an explicit bounded verifier result proves a mismatch. SSH failures,
+ * timeouts and unexpected stderr do not establish anything about cached bytes. */
+export function assertRemoteProviderPackVerificationResult(
+  result: Pick<Awaited<ReturnType<CommandManagedRuntimeRunner["execute"]>>, "exitCode" | "timedOut" | "stdout" | "stderr">,
+  expectedOpenCodeVersion?: string,
+): void {
+  if (result.timedOut) throw new RemoteProviderPackVerificationError("unavailable", "timeout", result);
+  if (result.exitCode === 0) {
+    if (expectedOpenCodeVersion !== undefined && result.stdout.trim() !== expectedOpenCodeVersion)
+      throw new RemoteProviderPackVerificationError("mismatch", "opencode_version_mismatch", result);
+    return;
+  }
+  if (result.exitCode === 42 && expectedOpenCodeVersion === undefined) {
+    for (const reason of providerPackMismatchReasons) {
+      if (result.stderr.trim() === `paperclip-provider-pack-verification:${reason}`)
+        throw new RemoteProviderPackVerificationError("mismatch", reason, result);
+    }
+    for (const reason of providerPackCompatibilityReasons) {
+      if (result.stderr.trim() === `paperclip-provider-pack-verification:${reason}`)
+        throw new RemoteProviderPackVerificationError("incompatible", reason, result);
+    }
+  }
+  throw new RemoteProviderPackVerificationError("unavailable", "command_failed", result);
+}
+
 /** The controller's retained attachment template pins content-addressed paths.
  * Remote manifests may supply bytes, but cannot choose a new code authority. */
 export function pinnedComputerProviderPack(input: {
@@ -232,7 +267,12 @@ export async function prepareComputerProviderPackCache(input: {
   };
   const verifyShared = async () => {
     try { await input.verify(input.cacheRoot); }
-    catch (cause) { throw new Error(`runner_remote_provider_cache_corrupt: ${input.cacheRoot}; stop tasks using this pack and have an operator remove or quarantine this exact cache directory before retrying`, { cause }); }
+    catch (cause) {
+      if (cause instanceof RemoteProviderPackVerificationError && cause.kind === "mismatch")
+        throw new Error(`runner_remote_provider_cache_corrupt: ${cause.reason}; ${input.cacheRoot}; stop tasks using this pack and have an operator remove or quarantine this exact cache directory before retrying`, { cause });
+      if (cause instanceof RemoteProviderPackVerificationError) throw cause;
+      throw new RemoteProviderPackVerificationError("unavailable", "transport_error", cause);
+    }
   };
   if (await operation("exists") === "true") {
     await verifyShared();