mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 13:24:55 +01:00
fix: reuse verified provider packs across equivalent builds
Normalize the provider-pack layout independently of runnerd build order and compare verified content rather than revision provenance, avoiding unnecessary multi-gigabyte uploads. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
a31eba2381
commit
d3257a3340
8 files changed
+235
-29
No files matched your search
@@ -12,8 +12,9 @@ import {
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
heartbeatRuns,
|
||||
@@ -26,7 +27,7 @@ import {
|
||||
type NativeExecutionInputV1,
|
||||
type PrpEvent,
|
||||
} from "@paperclipai/paperclip-runner";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
createNativeHarnessBackupStamp,
|
||||
verifyNativeHarnessBackupStamp,
|
||||
@@ -192,6 +193,7 @@ import {
|
||||
normalizeNativeUsage,
|
||||
parseRemoteRunnerProcessIdentity,
|
||||
readRemoteProviderPackManifest,
|
||||
buildRemoteProviderPackVerificationScript,
|
||||
providerSessionIdentityFromDurableProviderState,
|
||||
providerSessionIdentityTransitionIsAllowed,
|
||||
providerPlanMarkdown,
|
||||
@@ -589,6 +591,12 @@ describe("native provider usage normalization", () => {
|
||||
});
|
||||
|
||||
describe("remote provider pack manifest", () => {
|
||||
it("normalizes app and sandbox build layouts in the standard server test lane", () => {
|
||||
execFileSync(process.execPath, ["--test", fileURLToPath(new URL(
|
||||
"../../../../packages/paperclip-runner/scripts/provider-pack-layout.test.mjs", import.meta.url,
|
||||
))], { stdio: "pipe" });
|
||||
});
|
||||
|
||||
const canonical = (value: unknown): string => {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
@@ -648,7 +656,7 @@ describe("remote provider pack manifest", () => {
|
||||
pi: "0.84.2",
|
||||
piAcp: "0.0.33",
|
||||
},
|
||||
target: { platform: "linux", architecture: "x64" },
|
||||
target: { platform: process.platform, architecture: process.arch },
|
||||
runnerSourceRevision: "1".repeat(40),
|
||||
distDigest: sha256DirectoryTree(join(root, "dist")),
|
||||
bridgeDigest: "",
|
||||
@@ -703,6 +711,116 @@ describe("remote provider pack manifest", () => {
|
||||
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
|
||||
"1.18.29",
|
||||
);
|
||||
// Execute the exact source sent to the sandbox, not a mock of its verdict.
|
||||
const expectedManifest = readRemoteProviderPackManifest(root);
|
||||
const expectedArgument = Buffer.from(canonical(expectedManifest)).toString("base64");
|
||||
const verifyRemote = () => spawnSync(process.execPath, [
|
||||
"-e", buildRemoteProviderPackVerificationScript(), root, expectedArgument,
|
||||
], { encoding: "utf8" });
|
||||
for (const [pkg, version] of Object.entries({
|
||||
acpx: payload.pins.acpx,
|
||||
"@agentclientprotocol/claude-agent-acp": payload.pins.claudeAcp,
|
||||
"@agentclientprotocol/codex-acp": payload.pins.codexAcp,
|
||||
"opencode-ai": payload.pins.opencode,
|
||||
})) {
|
||||
await mkdir(join(root, "node_modules", pkg), { recursive: true });
|
||||
await writeFile(join(root, "node_modules", pkg, "package.json"), JSON.stringify({ version }));
|
||||
}
|
||||
expect(verifyRemote().status).toBe(0);
|
||||
payload.runnerSourceRevision = "2".repeat(40);
|
||||
await writeManifest();
|
||||
expect(verifyRemote().status).toBe(0); // Revision-only provenance allows image reuse.
|
||||
|
||||
// Exercise the production selection branch too: a different revision with
|
||||
// verified identical bytes must link the installed pack without syncIn.
|
||||
payload.runnerSourceRevision = "1".repeat(40);
|
||||
await writeManifest();
|
||||
const syncIn = vi.fn(async () => { throw new Error("unexpected-provider-pack-upload"); });
|
||||
const logs: string[] = [];
|
||||
const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => {
|
||||
let stdout = "";
|
||||
const script = command.args?.[1] ?? "";
|
||||
if (command.args?.[0] === "--build-metadata") {
|
||||
stdout = JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
capabilities: ["codex.warm-attachment.passive-notices.v1"], prpTransportModes: ["listen_ws"],
|
||||
});
|
||||
} else if (script.includes("command -v paperclip-runnerd")) {
|
||||
stdout = "/opt/paperclip-runner/bin/paperclip-runnerd\n";
|
||||
} else if (script.includes("for candidate in /opt/paperclip-runner/provider-pack")) {
|
||||
stdout = "/opt/paperclip-runner/provider-pack\n";
|
||||
} else if (command.args?.[0] === "-e") {
|
||||
const verified = spawnSync(process.execPath, ["-e", script, root, command.args![3]!], { encoding: "utf8" });
|
||||
return { exitCode: verified.status, signal: null, timedOut: false, stdout: verified.stdout, stderr: verified.stderr };
|
||||
} else if (command.command.endsWith("/node_modules/.bin/opencode") && command.args?.[0] === "--version") {
|
||||
stdout = payload.pins.opencode;
|
||||
} else if (!script.includes("ln -s")) {
|
||||
throw new Error("after-provider-pack-verification");
|
||||
}
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout, stderr: "" };
|
||||
});
|
||||
const providerExecution = {
|
||||
...execution,
|
||||
provider: { kind: "opencode", model: null },
|
||||
session: { ...execution.session, normalizedSessionId: `revision-only-${randomUUID()}`, driverKind: "opencode_server" },
|
||||
} as NativeExecutionInputV1;
|
||||
await createRunnerdBackend({
|
||||
db: leaseDb(providerExecution), execution: providerExecution,
|
||||
runnerInstanceId: "runner-revision-only-provider-pack", runnerIngressAuthorized: true,
|
||||
runnerRemoteProviderPackPath: root,
|
||||
onLog: async (_stream, chunk) => { logs.push(chunk); },
|
||||
runnerExecutionTarget: {
|
||||
kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
|
||||
leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
|
||||
runner: { execute: remoteExecute, syncIn },
|
||||
} as never,
|
||||
});
|
||||
payload.runnerSourceRevision = "2".repeat(40);
|
||||
await writeManifest();
|
||||
state.createTransport.mockClear();
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const transport = state.createTransport.mock.calls[0]![0] as RunnerTransportOptions & {
|
||||
controlPlaneRegistration: (authority: unknown) => Promise<unknown>;
|
||||
};
|
||||
await expect(transport.controlPlaneRegistration({})).rejects.toThrow();
|
||||
expect(logs.join("")).toContain("using content-matched provider pack");
|
||||
expect(syncIn).not.toHaveBeenCalled();
|
||||
const revisionManifest = JSON.parse(await readFile(join(root, "provider-pack.json"), "utf8"));
|
||||
await writeFile(join(root, "provider-pack.json"), JSON.stringify({ ...revisionManifest, digest: `sha256:${"0".repeat(64)}` }));
|
||||
expect(verifyRemote().stderr).toContain("manifest digest mismatch");
|
||||
await writeManifest();
|
||||
for (const [relativePath, bytes] of [
|
||||
["node_modules/node/bin/node", node],
|
||||
["pnpm-lock.yaml", lockfile],
|
||||
["dist/cli/acpx-runtime-sidecar.cjs", sidecar],
|
||||
]) {
|
||||
await writeFile(join(root, relativePath), "tampered bytes");
|
||||
expect(verifyRemote().status).not.toBe(0);
|
||||
await writeFile(join(root, relativePath), bytes);
|
||||
}
|
||||
await writeFile(join(root, "dist", "extra-runtime.js"), "unexpected runtime code");
|
||||
expect(verifyRemote().stderr).toContain("dist tree digest mismatch");
|
||||
await rm(join(root, "dist", "extra-runtime.js"));
|
||||
for (const change of [
|
||||
(value: typeof payload) => { value.pins.opencode = "0.0.0"; },
|
||||
(value: typeof payload) => { value.target.architecture = "unexpected" as typeof process.arch; },
|
||||
(value: typeof payload) => { value.artifacts.nodeCommand = { ...value.artifacts.productionLock }; },
|
||||
]) {
|
||||
const changed = structuredClone(payload);
|
||||
change(changed);
|
||||
await writeFile(join(root, "provider-pack.json"), JSON.stringify({
|
||||
schema: expectedManifest.schema,
|
||||
digest: digest(canonical(changed)), payload: changed,
|
||||
}));
|
||||
expect(verifyRemote().stderr).toContain("manifest content mismatch");
|
||||
}
|
||||
await writeManifest();
|
||||
await writeFile(join(root, "node_modules", "acpx", "package.json"), JSON.stringify({ version: "0.0.0" }));
|
||||
expect(verifyRemote().stderr).toContain("acpx version mismatch");
|
||||
await writeFile(join(root, "node_modules", "acpx", "package.json"), JSON.stringify({ version: payload.pins.acpx }));
|
||||
expect(verifyRemote().status).toBe(0);
|
||||
|
||||
for (const [artifactName, substituteName] of [
|
||||
["nodeCommand", "productionLock"],
|
||||
["opencodeExecutable", "opencodeCommand"],
|
||||
|
||||
@@ -5341,6 +5341,36 @@ export function readRemoteProviderPackManifest(
|
||||
return structuredClone(manifest);
|
||||
}
|
||||
|
||||
/** Verify bytes against the host-owned content contract while retaining revision provenance. */
|
||||
export function buildRemoteProviderPackVerificationScript(): string {
|
||||
return [
|
||||
"const fs=require('node:fs')",
|
||||
"const crypto=require('node:crypto')",
|
||||
"const path=require('node:path')",
|
||||
"const root=process.argv[1]",
|
||||
"const expected=JSON.parse(Buffer.from(process.argv[2],'base64').toString('utf8'))",
|
||||
"const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
|
||||
"const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
|
||||
"const manifest=JSON.parse(actual)",
|
||||
"const digest=(payload)=>'sha256:'+crypto.createHash('sha256').update(canonical(payload)).digest('hex')",
|
||||
"if(manifest.schema!==expected.schema||!manifest.payload||!(/^[0-9a-f]{40}(?:-dirty)?$/).test(manifest.payload.runnerSourceRevision))throw new Error('manifest schema or revision mismatch')",
|
||||
"if(digest(manifest.payload)!==manifest.digest)throw new Error('manifest digest mismatch')",
|
||||
"const content=(value)=>{const {digest:provenanceDigest,...fields}=value;const {runnerSourceRevision,...payload}=fields.payload;return {...fields,payload}}",
|
||||
"if(canonical(content(manifest))!==canonical(content(expected)))throw new Error('manifest content mismatch')",
|
||||
"const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
|
||||
"const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
|
||||
"for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
|
||||
"if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
|
||||
"const version=process.versions.node.split('.').map(Number)",
|
||||
"const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)",
|
||||
"if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')",
|
||||
"if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
|
||||
"const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
|
||||
"const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
|
||||
"for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
|
||||
].join(";");
|
||||
}
|
||||
|
||||
export function assertRemoteRunnerBuildMetadata(
|
||||
value: unknown,
|
||||
requiredMode: "dial_wss" | "listen_ws",
|
||||
@@ -6544,28 +6574,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
packRoot,
|
||||
expectedProviderPackManifest.payload.artifacts.nodeCommand.path,
|
||||
);
|
||||
const verifyScript = [
|
||||
"const fs=require('node:fs')",
|
||||
"const crypto=require('node:crypto')",
|
||||
"const path=require('node:path')",
|
||||
"const root=process.argv[1]",
|
||||
"const expected=Buffer.from(process.argv[2],'base64').toString('utf8')",
|
||||
"const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
|
||||
"const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
|
||||
"const manifest=JSON.parse(actual)",
|
||||
"if(canonical(manifest)!==expected)throw new Error('manifest mismatch')",
|
||||
"const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
|
||||
"const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
|
||||
"for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
|
||||
"if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
|
||||
"const version=process.versions.node.split('.').map(Number)",
|
||||
"const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)",
|
||||
"if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')",
|
||||
"if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
|
||||
"const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
|
||||
"const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
|
||||
"for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
|
||||
].join(";");
|
||||
const verifyScript = buildRemoteProviderPackVerificationScript();
|
||||
const verified = await remoteCommandRunner.execute({
|
||||
command: providerNodeCommand,
|
||||
args: ["-e", verifyScript, packRoot, expected],
|
||||
@@ -6905,7 +6914,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
|
||||
await input.onLog?.(
|
||||
"stderr",
|
||||
"[paperclip-runner] using manifest-matched provider pack from the sandbox image\n",
|
||||
"[paperclip-runner] using content-matched provider pack from the sandbox image\n",
|
||||
);
|
||||
} catch {
|
||||
preinstalledProviderPack = null;
|
||||
|
||||
Reference in new issue
Block a user