diff --git a/server/src/services/native-runtime/native-run-trace.test.ts b/server/src/services/native-runtime/native-run-trace.test.ts index 8954fe10442..47e2e44875a 100644 --- a/server/src/services/native-runtime/native-run-trace.test.ts +++ b/server/src/services/native-runtime/native-run-trace.test.ts @@ -69,6 +69,18 @@ function createRecordingTraceContext(): { } describe("native runner performance trace", () => { + it("records the named check of a failed measurement on its run event", async () => { + const {traceContext}=createRecordingTraceContext(); + const events: Array> = []; + const trace=createNativeRunTrace({runId:"provider-pack-check",startedAtMs:1000,traceContext, + onEvent:async event=>{events.push(event.payload as Record);}}); + const rejected=Object.assign(new Error("runner_remote_provider_artifact_incompatible: incompatible"),{failureCheck:"contract_digest"}); + await expect(trace.measure("provider_pack.compatibility",async()=>{throw rejected;},{attributes:{target:"image"}})).rejects.toBe(rejected); + await expect(trace.measure("provider_pack.verify",async()=>{throw Object.assign(new Error("x"),{failureCheck:"has space"});})).rejects.toThrow("x"); + await trace.finish("ok"); + expect(events.find(event=>event.span==="provider_pack.compatibility")).toMatchObject({outcome:"failed",target:"image",check:"contract_digest"}); + expect(events.find(event=>event.span==="provider_pack.verify")).not.toHaveProperty("check"); + }); it("projects a full durable diagnostic batch without waiting on duplicate run-log writes", async () => { const {traceContext, spans}=createRecordingTraceContext(); let release!: () => void; diff --git a/server/src/services/native-runtime/native-run-trace.ts b/server/src/services/native-runtime/native-run-trace.ts index 9911b158cd0..2e8f5c54676 100644 --- a/server/src/services/native-runtime/native-run-trace.ts +++ b/server/src/services/native-runtime/native-run-trace.ts @@ -210,6 +210,7 @@ const SAFE_NATIVE_RUN_SPAN_ATTRIBUTE_KEYS = new Set([ "adapter", "anchor", "bytesTransferred", + "check", "inputBytes", "toolSchemaBytes", "toolCount", "contextReused", "tokenDataAvailable", "connectionOwner", "driver", @@ -229,6 +230,11 @@ const SAFE_NATIVE_RUN_SPAN_ATTRIBUTE_KEYS = new Set([ "target", ]); +export function nativeRunFailureCheck(error: unknown): string | undefined { + const check = (error as { failureCheck?: unknown } | null)?.failureCheck; + return typeof check === "string" && /^[A-Za-z0-9_.:,@/-]{1,120}$/.test(check) ? check : undefined; +} + function finiteMs(value: number): number { return Number.isFinite(value) ? Math.max(0, value) : 0; } @@ -630,6 +636,8 @@ export function createNativeRunTrace(input: { return await run(scope, fn); } catch (error) { outcome = "failed"; + const check = nativeRunFailureCheck(error); + if (check) annotate(scope, { check }); throw error; } finally { await end(scope, { outcome }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 26badf5ba56..0fa234824a3 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -50,7 +50,7 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; import { nativeProviderPackAdmissionFailureCode, prepareNativeProviderPackAdmission, rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack, recordNativeProviderPackAdmission, type NativeProviderPackAdmissionSource } from "./native-provider-pack-admission.js"; -import { type ProviderPackInstallation, verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js"; +import { type ProviderPackInstallation, providerPackIncompatible, verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, lockNativeExecutionLeaseForReacquire, nativeExecutionLeaseHeldCondition, releaseUnstartedNativeExecutionClaim, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; export { renewNativeSessionExecutionLease } from "./native-execution-lease.js"; @@ -14706,7 +14706,8 @@ export async function prepareRemoteProviderPackOnTarget(input: { "provider_pack.verify", () => verifyRemoteProviderPack(stagedRemoteProviderPackRoot), ), - usePreinstalled: async () => useVerifiedPreinstalledRemoteProviderPack({ + usePreinstalled: async (onRejected) => useVerifiedPreinstalledRemoteProviderPack({ + onRejected, candidates: preinstalledProviderPackRoots = await measureNativeRunnerSpan(input.trace, "provider_pack.discover", discoverPreinstalledProviderPack), verifyAndLink: async (preinstalledProviderPack) => { await measureNativeRunnerSpan( @@ -14736,9 +14737,7 @@ export async function prepareRemoteProviderPackOnTarget(input: { timeoutMs: 10_000, })); if (linked.exitCode !== 0 || linked.timedOut) { - throw new Error( - "runner_remote_provider_artifact_incompatible: preinstalled provider pack could not be linked", - ); + throw providerPackIncompatible("link_failed", "preinstalled provider pack could not be linked"); } activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot; await input.onLog?.( @@ -17151,7 +17150,7 @@ async function createRunnerdBackendWithinSessionClaim( requireExactIdentity: retainProviderPack || explicitProviderPackSelection, requireCurrentOpenCodeProxy: !retainProviderPack && runnerEnvironment?.PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS === "1", - })); + }), { attributes: { target: installation.root === stagedRemoteProviderPackRoot ? "workspace" : "image" } }); if (!retainProviderPack) providerPackAuthority!.select(manifest); expectedProviderPackManifest = manifest; activeRemoteProviderPackRoot = installation.root; diff --git a/server/src/services/native-runtime/remote-provider-pack-verification.test.ts b/server/src/services/native-runtime/remote-provider-pack-verification.test.ts index 3c4255a4dc5..308d142b9cc 100644 --- a/server/src/services/native-runtime/remote-provider-pack-verification.test.ts +++ b/server/src/services/native-runtime/remote-provider-pack-verification.test.ts @@ -51,8 +51,20 @@ describe("installed provider pack compatibility", () => { try { // Exercise the old complete-manifest comparison in a real Node process. // Only provenance changed; provider bytes and interface are identical. - await expect(verifyRemoteProviderPack(input)).rejects.toThrow("manifest mismatch"); + await expect(verifyRemoteProviderPack(input)).rejects.toMatchObject({ + message: expect.stringContaining("manifest mismatch"), failureCheck: "manifest_mismatch:runnerSourceRevision" }); expect(execute).toHaveBeenCalledOnce(); + await expect(verifyRemoteProviderPack({ ...input, manifest: installed })).rejects.toMatchObject({ + message: expect.stringContaining("nodeCommand digest mismatch"), failureCheck: "artifact_digest:nodeCommand" }); + await rm(join(root, "provider-pack.json")); + await expect(verifyRemoteProviderPack(input)).rejects.toMatchObject({ failureCheck: "manifest_unreadable" }); + const combined = vi.fn(async (request: Parameters[0]) => { + const result = await execute(request); + return { ...result, stdout: `${result.stdout}${result.stderr}`, stderr: "" }; + }); + await expect(verifyRemoteProviderPack({ ...input, runner: { execute: combined } })) + .rejects.toMatchObject({ failureCheck: "manifest_unreadable" }); + await writeFile(join(root, "provider-pack.json"), JSON.stringify(installed)); execute.mockClear(); const selected = await verifyRemoteProviderPack({ ...input, installation: { root, manifest: installed, immutable: true }, provider: { kind: "opencode" } }); @@ -70,6 +82,31 @@ describe("installed provider pack compatibility", () => { provider: { kind: "opencode" }, requireExactIdentity: options.requireExactIdentity, requireCurrentOpenCodeProxy: options.requireCurrentOpenCodeProxy }); }; + it.each([ + ["interface", "compatibility_revision"], ["contracts", "contract_digest"], ["platform", "target"], ["schema", "schema"], + ["artifact_path", "artifact:nodeCommand"], ["digest", "manifest_digest"], ["legacy", "compatibility_revision"], + ["legacy_contracts", "contract_digest_missing"], + ])("names the failed check for an incompatible %s", async (field, failureCheck) => { + let pack = manifest("a"); + if (field === "interface") pack.payload.compatibility!.opencode += 1; + if (field === "contracts") pack.payload.contractDigest = sha("other-contracts"); + if (field === "legacy") delete pack.payload.compatibility; + if (field === "legacy_contracts") delete pack.payload.contractDigest; + if (field === "platform") pack.payload.target.architecture = "arm64"; + if (field === "schema") pack.schema = "unknown"; + if (field === "artifact_path") pack.payload.artifacts.nodeCommand.path = "../node"; + pack = reseal(pack); + if (field === "digest") pack.digest = sha("corrupt"); + await expect(check(pack)).rejects.toMatchObject({ failureCheck }); + }); + it("names unprotected, exact-identity and proxy rejections", async () => { + await expect(check(manifest("b"), { immutable: false })).rejects.toMatchObject({ failureCheck: "installation_unprotected" }); + await expect(check(manifest("a"), { requireExactIdentity: true })).rejects.toMatchObject({ failureCheck: "exact_identity" }); + await expect(check(manifest("a"), { requireCurrentOpenCodeProxy: true })).rejects.toMatchObject({ failureCheck: "opencode_proxy" }); + const unpinned = manifest("b"); + (unpinned.payload.pins as Record).opencode = null; + await expect(check(reseal(unpinned))).rejects.toMatchObject({ failureCheck: "opencode_pin_missing" }); + }); it.each(["interface", "contracts", "platform", "schema", "artifact_path", "digest", "legacy", "legacy_contracts"])("rejects an incompatible %s", async field => { let pack = manifest("a"); if (field === "interface") pack.payload.compatibility!.opencode += 1; diff --git a/server/src/services/native-runtime/remote-provider-pack-verification.ts b/server/src/services/native-runtime/remote-provider-pack-verification.ts index 5d48ea89146..5e7c424f20b 100644 --- a/server/src/services/native-runtime/remote-provider-pack-verification.ts +++ b/server/src/services/native-runtime/remote-provider-pack-verification.ts @@ -28,6 +28,22 @@ export type ProviderPackIdentity = { }; }; +export function providerPackIncompatible(check: string, detail: string): Error & { failureCheck: string } { + return Object.assign(new Error(`runner_remote_provider_artifact_incompatible: ${detail}`), { failureCheck: check }); +} + +async function withProviderPackCheck(step: "pack" | "opencode", verify: () => Promise): Promise { + try { + return await verify(); + } catch (error) { + if (!(error instanceof Error) || error.name === "AbortError") throw error; + const reported = /paperclip-pack-check=([A-Za-z0-9_.:,@/-]{1,100})/.exec(error.message)?.[1]; + const check = reported ?? (error.message.startsWith("runner_remote_provider_verification_unavailable") + ? `${step}_verifier_unavailable` : `${step}_verifier_failed`); + throw Object.assign(error, { failureCheck: check }); + } +} + export type ProviderPackInstallation = { root: string; manifest: unknown; immutable: boolean }; export type PackProvider = { kind: "opencode" } | { kind: "acpx"; agent: string }; @@ -45,28 +61,33 @@ function compatibleInstallation( const actual = installation.manifest as T | null; const payload = actual?.payload; const contract = expected.payload.compatibility?.[provider.kind]; - if (!installation.immutable || installation.root !== packRoot || !payload - || actual.schema !== expected.schema - || actual.digest !== `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` - || canonicalProviderPackJson(payload.target) !== canonicalProviderPackJson(expected.payload.target) - || (requireExactIdentity ? actual.digest !== expected.digest - : !Number.isSafeInteger(contract) || contract! < 1 || payload.compatibility?.[provider.kind] !== contract - || !/^sha256:[a-f0-9]{64}$/.test(expected.payload.contractDigest ?? "") - || payload.contractDigest !== expected.payload.contractDigest) - || typeof payload.pins?.opencode !== "string") { - throw new Error("runner_remote_provider_artifact_incompatible: installed provider pack interface or installation is incompatible"); + const failed = !installation.immutable ? "installation_unprotected" + : installation.root !== packRoot ? "installation_root" + : !payload ? "manifest_missing" + : actual.schema !== expected.schema ? "schema" + : actual.digest !== `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` ? "manifest_digest" + : canonicalProviderPackJson(payload.target) !== canonicalProviderPackJson(expected.payload.target) ? "target" + : requireExactIdentity ? (actual.digest !== expected.digest ? "exact_identity" : null) + : !Number.isSafeInteger(contract) || contract! < 1 ? "expected_compatibility_missing" + : payload.compatibility?.[provider.kind] !== contract ? "compatibility_revision" + : !/^sha256:[a-f0-9]{64}$/.test(expected.payload.contractDigest ?? "") ? "expected_contract_digest_missing" + : payload.contractDigest === undefined ? "contract_digest_missing" + : payload.contractDigest !== expected.payload.contractDigest ? "contract_digest" + : null; + if (failed || !actual || !payload || typeof payload.pins?.opencode !== "string") { + throw providerPackIncompatible(failed ?? "opencode_pin_missing", "installed provider pack interface or installation is incompatible"); } // Launch profiles and recovery receipts bind the selected installation's own // bytes, never the current controller's artifact hashes. for (const [name, expectedArtifact] of Object.entries(expected.payload.artifacts)) { const artifact = payload.artifacts?.[name]; if (artifact?.path !== expectedArtifact.path || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256)) { - throw new Error("runner_remote_provider_artifact_incompatible: installed provider pack artifact identity is invalid"); + throw providerPackIncompatible(`artifact:${name}`, "installed provider pack artifact identity is invalid"); } } if (provider.kind === "opencode" && requireCurrentOpenCodeProxy && payload.artifacts.opencodeProxy?.sha256 !== expected.payload.artifacts.opencodeProxy?.sha256) { - throw new Error("runner_remote_provider_artifact_incompatible: installed proxy does not match the requested OpenCode diagnostics"); + throw providerPackIncompatible("opencode_proxy", "installed proxy does not match the requested OpenCode diagnostics"); } if (provider.kind === "acpx") { const agent = provider.agent; @@ -78,7 +99,7 @@ function compatibleInstallation( if (currentProfile ? payload.acpxProfileDigests?.[agent] !== currentProfile : !selected || !installed || ["version", "profileDigest", "closureDigest", "qualification", "path"].some(key => (installed as Record)[key] !== (selected as Record)[key])) { - throw new Error("runner_remote_provider_artifact_incompatible: selected ACPX provider profile is incompatible"); + throw providerPackIncompatible("acpx_profile", "selected ACPX provider profile is incompatible"); } } return structuredClone(actual); @@ -112,47 +133,48 @@ export async function verifyRemoteProviderPack({ "const path=require('node:path')", "const root=process.argv[1]", "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')", - "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()", + "const fail=(check,detail)=>{fs.writeSync(2,detail+'\\n');fs.writeSync(1,'\\npaperclip-pack-check='+check+'\\n');process.exit(3)}", + "let actual;try{actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()}catch{fail('manifest_unreadable','provider pack manifest unreadable')}", "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)", - "const manifest=JSON.parse(actual)", - "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')", - "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')", - "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", - "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}", - "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')", - "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}", + "let manifest;try{manifest=JSON.parse(actual)}catch{fail('manifest_invalid','provider pack manifest invalid')}", + "const differing=(a,b)=>[...new Set([...Object.keys(a||{}),...Object.keys(b||{})])].sort().filter(k=>canonical((a||{})[k])!==canonical((b||{})[k]))", + "if(canonical(manifest)!==expected){const wanted=JSON.parse(expected);const keys=differing(manifest,wanted).flatMap(k=>k==='payload'?differing(manifest.payload,wanted.payload):k==='digest'?[]:[k]);fail('manifest_mismatch:'+keys.join(',').slice(0,90),'manifest mismatch')}", + "const hash=(p)=>{try{return 'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')}catch{return 'unreadable'}}", + "const walk=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", + "const tree=(treeRoot)=>{try{return walk(treeRoot)}catch{return 'unreadable'}}", + "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)fail('artifact_digest:'+name,name+' digest mismatch')}", + "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)fail('dist_digest','dist tree digest mismatch')", + "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)fail('candidate_digest','candidate asset tree digest mismatch')}", "const version=process.versions.node.split('.').map(Number)", "const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)", - "if(version[0]JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version", + "if(version[0]{try{return JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version}catch{return null}}", "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}", - "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')", + "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)fail('package_version:'+pkg,pkg+' version mismatch')", ].join(";"); - await runRemoteProviderVerification(() => runner.execute({ + await withProviderPackCheck("pack", () => runRemoteProviderVerification(() => runner.execute({ command: providerNodeCommand, args: ["-e", verifyScript, packRoot, expected], cwd: cwd, bypassSession: true, timeoutMs: 30_000, - }), "provider pack verification"); + }), "provider pack verification")); const opencodeCommand = posix.join( packRoot, manifest.payload.artifacts.opencodeCommand.path, ); - const opencodeVersion = await runRemoteProviderVerification(() => runner.execute({ + const opencodeVersion = await withProviderPackCheck("opencode", () => runRemoteProviderVerification(() => runner.execute({ command: opencodeCommand, args: ["--version"], cwd: cwd, bypassSession: true, timeoutMs: 30_000, - }), "OpenCode version verification"); + }), "OpenCode version verification")); if ( opencodeVersion.stdout.trim() !== manifest.payload.pins.opencode ) { - throw new Error( - "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", - ); + throw providerPackIncompatible("opencode_version", "OpenCode version mismatch"); } return structuredClone(manifest); } diff --git a/server/src/services/native-runtime/remote-provider-pack.test.ts b/server/src/services/native-runtime/remote-provider-pack.test.ts index 5d5991abca4..0ae23290c7f 100644 --- a/server/src/services/native-runtime/remote-provider-pack.test.ts +++ b/server/src/services/native-runtime/remote-provider-pack.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { createNativeRunTrace } from "./native-run-trace.js"; import { prepareVerifiedRemoteProviderPack, useVerifiedPreinstalledRemoteProviderPack, runRemoteProviderVerification, shouldUseBundledAcpxImageAssets } from "./remote-provider-pack.js"; describe("remote provider verification", () => { @@ -96,6 +97,30 @@ describe("remote provider pack reuse", () => { stageAndVerify.mockResolvedValue(undefined); await expect(prepareVerifiedRemoteProviderPack(input)).resolves.toBe("uploaded"); }); + + it("records on the selection span why the image pack was not reused", async () => { + const trace = createNativeRunTrace({ runId: "provider-pack-select-run", startedAtMs: Date.now() }); + const annotate = vi.spyOn(trace, "annotate"); + const rejected = Object.assign(new Error("runner_remote_provider_artifact_incompatible: provider pack verification failed"), + { failureCheck: "manifest_mismatch:runnerSourceRevision" }); + await expect(prepareVerifiedRemoteProviderPack({ + trace, + verifyStaged: async () => { throw new Error("missing"); }, + usePreinstalled: onRejected => useVerifiedPreinstalledRemoteProviderPack({ + candidates: ["/opt/paperclip-runner/provider-pack", "/home/operator/.local/share/paperclip-runner/provider-pack"], onRejected, + verifyAndLink: async root => { + throw root.startsWith("/opt/") ? rejected : Object.assign(new Error("missing"), { failureCheck: "manifest_unreadable" }); + }, + }), + stageAndVerify: async () => {}, + })).resolves.toBe("uploaded"); + expect(annotate).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + stateSource: "uploaded", reason: "cached_packs_unverified", check: "manifest_mismatch:runnerSourceRevision" })); + annotate.mockClear(); + await prepareVerifiedRemoteProviderPack({ trace, verifyStaged: async () => { throw new Error("missing"); }, + usePreinstalled: async () => false, stageAndVerify: async () => {} }); + expect(annotate).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ check: "no_preinstalled_pack" })); + }); }); diff --git a/server/src/services/native-runtime/remote-provider-pack.ts b/server/src/services/native-runtime/remote-provider-pack.ts index 15eb74ff0b5..b803d6d6d2c 100644 --- a/server/src/services/native-runtime/remote-provider-pack.ts +++ b/server/src/services/native-runtime/remote-provider-pack.ts @@ -1,5 +1,5 @@ import { redactSensitiveText } from "../../redaction.js"; -import type { NativeRunTrace } from "./native-run-trace.js"; +import { nativeRunFailureCheck, type NativeRunTrace } from "./native-run-trace.js"; import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; /** Qualified ACPX releases carry their Linux image manifest and daemon. */ @@ -55,24 +55,27 @@ export async function runRemoteProviderVerification( export async function prepareVerifiedRemoteProviderPack(input: { trace?: NativeRunTrace; verifyStaged: () => Promise; - usePreinstalled: () => Promise; + usePreinstalled: (onRejected: (check: string) => void) => Promise; stageAndVerify: () => Promise; }): Promise<"staged" | "preinstalled" | "uploaded"> { const scope = input.trace?.start("provider_pack.select"); let failed = false; + let preinstalledCheck = "no_preinstalled_pack"; const select = async () => { try { await input.verifyStaged(); return "staged" as const; } catch {} - if (await input.usePreinstalled()) return "preinstalled" as const; + let rejected = false; + if (await input.usePreinstalled(check => { if (!rejected) preinstalledCheck = check; rejected = true; })) return "preinstalled" as const; await input.stageAndVerify(); return "uploaded" as const; }; try { const source = scope ? await input.trace!.run(scope, select) : await select(); if (scope) input.trace!.annotate(scope, { stateSource: source, - reason: source === "uploaded" ? "cached_packs_unverified" : "verified_pack_reused" }); + reason: source === "uploaded" ? "cached_packs_unverified" : "verified_pack_reused", + ...(source === "uploaded" ? { check: preinstalledCheck } : {}) }); return source; } catch (error) { failed = true; @@ -85,12 +88,14 @@ export async function prepareVerifiedRemoteProviderPack(input: { export async function useVerifiedPreinstalledRemoteProviderPack(input: { candidates: readonly string[]; verifyAndLink: (root: string) => Promise; + onRejected?: (check: string) => void; }): Promise { for (const candidate of input.candidates) { try { await input.verifyAndLink(candidate); return true; - } catch { + } catch (error) { + input.onRejected?.(nativeRunFailureCheck(error) ?? "verification_failed"); } } return false;