diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index 759f5ef4c1..be447aeb7c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -928,6 +928,18 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec @@ -43,7 +45,10 @@ export function nativeCompletionGates(variant) { "retains the first provider_variance failure without a second attempt"] }, ]; } -export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery"]; +export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery", "NC-rust-carrier"]; +export function nativeCompletionCommandGateIds(mode) { + return NATIVE_COMPLETION_COMMAND_GATE_IDS.map(id => mode === "trusted_hosted_archive" && id === "NC-rust-carrier" ? "NC-hosted-runnerd" : id); +} export function gradeNativeCompletionGate(gate, report, exitCode) { const assertions = (report?.testResults ?? []).flatMap(file => file.assertionResults ?? []); const requirements = gate.required.map(name => ({ name, passed: assertions.some(assertion => @@ -66,19 +71,20 @@ export function gradeNativeCompletionDiscovery(output, exitCode) { rows.every(row => row.length === 6 && row[1] === "native-completion" && row[2] === "native" && row[4]?.trim() && row[5]?.trim()), executionIds: ids, expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 }; } +export function gradeNativeCompletionRustCarrier(output, exitCode) { + return exitCode === 0 && /^test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity \.\.\. ok$/m.test(output) && + /test result: ok\. 1 passed; 0 failed;/.test(output); +} export function nativeCompletionPrerequisiteEnvironment(source) { return Object.fromEntries(["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL", - "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]])); + "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "PAPERCLIP_RUNNER_E2E_SOURCE_SHA"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]])); } -function git(args) { return spawnSync("git", args, { cwd: root, encoding: "utf8" }); } function currentSource() { const source = nativeCompletionSourceFingerprint(); - const head = git(["rev-parse", "HEAD"]), ancestor = git(["merge-base", "--is-ancestor", source.baseSha, "HEAD"]); - const tracked = git(["ls-files", "--error-unmatch", "--", ...NATIVE_COMPLETION_SOURCE_FILES]); - const clean = git(["diff", "--quiet", "HEAD"]); - const status = git(["status", "--porcelain", "--untracked-files=normal"]); - return { ...source, sha: head.status === 0 ? head.stdout.trim() : null, - layering: ancestor.status === 0, immutable: tracked.status === 0 && clean.status === 0 && status.status === 0 && status.stdout.trim() === "" }; + return { ...source, ...inspectNativeCompletionSourceMetadata({ repositoryRoot: root, + sourceFiles: NATIVE_COMPLETION_SOURCE_FILES, baseSha: source.baseSha, variant: source.variant, + shallowParentAnchors: NATIVE_COMPLETION_SOURCE_CONTRACT.shallowParentAnchors, + }) }; } function buildOutputFingerprint() { const hash = createHash("sha256"); @@ -94,11 +100,25 @@ function buildOutputFingerprint() { if (!existsSync(join(root, directory, "index.js"))) throw new Error(`Missing prerequisite build output: ${directory}`); visit(directory); } + const daemon = runnerdBinary(); + const bytes = readFileSync(daemon); + hash.update(JSON.stringify(["debug/paperclip-runnerd", bytes.length])).update(bytes); return hash.digest("hex"); } +function runnerdBinary() { + return join(root, `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`); +} +function runnerdProvenance(source, env) { + return inspectNativeCompletionRunnerd({ repositoryRoot: root, sourceSha: source.sha, + sourceFingerprint: source.fingerprint, environment: env }); +} export function assertNativeCompletionPreflightReceipt(report, current) { - const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS]; - if (report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 || + const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...nativeCompletionCommandGateIds(current.runnerdProvenance?.mode)]; + const hostedGate = report?.gates?.find(gate => gate.id === "NC-hosted-runnerd"); + const truthfulHosted = current.runnerdProvenance?.mode !== "trusted_hosted_archive" || + hostedGate?.executed === false && hostedGate?.calibration === "not_executed" && hostedGate?.total === 0 && + hostedGate?.passedTests === 0 && hostedGate?.reuse === "trusted_same_run_build"; + if (!truthfulHosted || report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 || report.live !== "not_run" || report.sourceSha !== current.sha || !/^[a-f0-9]{40}$/.test(current.sha ?? "") || ![current.fingerprint, current.fixtureFingerprint, current.manifestFingerprint, current.buildOutputFingerprint] .every(value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) || @@ -106,11 +126,21 @@ export function assertNativeCompletionPreflightReceipt(report, current) { report.manifestFingerprint !== current.manifestFingerprint || report.variant !== current.variant || report.baseSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha || report.archiveSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha || report.layering !== true || current.layering !== true || report.immutable !== true || current.immutable !== true || + !/^[a-f0-9]{64}$/.test(current.sourceMetadataFingerprint ?? "") || + report.sourceMetadataFingerprint !== current.sourceMetadataFingerprint || + !Array.isArray(report.sourceMetadataErrors) || report.sourceMetadataErrors.length !== 0 || current.sourceMetadataErrors?.length !== 0 || !Array.isArray(report.sourceErrors) || report.sourceErrors.length !== 0 || current.sourceErrors?.length !== 0 || report.setup?.passed !== true || report.setup.sdkExitCode !== 0 || report.setup.runnerTypeScriptExitCode !== 0 || + report.setup.runnerdExitCode !== (current.runnerdProvenance?.mode === "trusted_hosted_archive" ? null : 0) || + current.runnerdProvenance?.passed !== true || !["trusted_hosted_archive", "fresh_local_build"].includes(current.runnerdProvenance?.mode) || + JSON.stringify(report.setup.runnerdProvenance) !== JSON.stringify(current.runnerdProvenance) || + report.setup.runnerdSelectedPath !== current.runnerdProvenance.selectedPath || + report.setup.runnerdSha256 !== current.runnerdProvenance.binarySha256 || + !/^[a-f0-9]{64}$/.test(current.runnerdProvenance.binarySha256 ?? "") || + report.setup.runnerdSourceSha !== current.sha || report.setup.runnerdSourceFingerprint !== current.fingerprint || report.setup.buildOutputFingerprint !== current.buildOutputFingerprint || - !Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 2 || - ["setup-sdk.txt", "setup-runner-typescript.txt"].some(file => report.setup.evidence.filter(row => + !Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 3 || + ["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].some(file => report.setup.evidence.filter(row => row.file === file && /^[a-f0-9]{64}$/.test(row.sha256 ?? "")).length !== 1) || !Array.isArray(report.gates) || report.gates.length !== expected.length || expected.some(id => report.gates.filter(gate => gate.id === id && gate.passed === true && gate.exitCode === 0).length !== 1) || @@ -141,7 +171,8 @@ function manifestCommands(env) { export function main(args = process.argv.slice(2)) { if (args.includes("--list")) { console.log(JSON.stringify({ variants: ["candidate", "historical"], gates: nativeCompletionGates("candidate"), - commands: NATIVE_COMPLETION_COMMAND_GATE_IDS, cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return; + commands: { local: nativeCompletionCommandGateIds("fresh_local_build"), hosted: nativeCompletionCommandGateIds("trusted_hosted_archive") }, + cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return; } if (args.some(arg => !arg.startsWith("--output-dir=") && !arg.startsWith("--verify="))) throw new Error("Use --list, --output-dir= or --verify=; never paid providers."); @@ -149,7 +180,7 @@ export function main(args = process.argv.slice(2)) { const verify = args.find(arg => arg.startsWith("--verify="))?.slice("--verify=".length); if (verify) { const report = assertNativeCompletionPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), { - ...source, buildOutputFingerprint: buildOutputFingerprint(), + ...source, buildOutputFingerprint: buildOutputFingerprint(), runnerdProvenance: runnerdProvenance(source, env), }); const output = resolve(verify, ".."); for (const evidence of report.setup.evidence) assertRetainedNativeCompletionEvidence(output, evidence); @@ -158,12 +189,18 @@ export function main(args = process.argv.slice(2)) { const grade = gradeNativeCompletionGate(gate, JSON.parse(assertRetainedNativeCompletionEvidence(output, retained.evidence)), 0); if (!grade.passed) throw new Error(`Missing or failed retained native prerequisite assertions: ${gate.id}`); } - for (const id of NATIVE_COMPLETION_COMMAND_GATE_IDS) { + for (const id of nativeCompletionCommandGateIds(report.setup.runnerdProvenance.mode)) { const retained = report.gates.find(row => row.id === id), text = assertRetainedNativeCompletionEvidence(output, retained.evidence); if (id === "NC-node" && (!/# fail 0\b/.test(text) || !/# tests [1-9]\d*\b/.test(text))) throw new Error("Missing passing native admission calibrations."); if (id === "NC-discovery" && !gradeNativeCompletionDiscovery(text.split("\n\n")[0], 0).passed) throw new Error("Native completion six-cell discovery changed."); + if (id === "NC-rust-carrier" && !gradeNativeCompletionRustCarrier(text, 0)) + throw new Error("Missing passing retained Rust terminal-tool carrier calibration."); + if (id === "NC-hosted-runnerd" && (retained.executed !== false || retained.calibration !== "not_executed" || + retained.total !== 0 || retained.passedTests !== 0 || + JSON.stringify(JSON.parse(text.split("\n\n")[0])) !== JSON.stringify(report.setup.runnerdProvenance))) + throw new Error("Hosted runnerd reuse must retain exact binary proof and report Rust calibration not executed."); if (id === "NC-manifest" && manifestCommands(env).some(run => run.status !== 0)) throw new Error("Generated native capability manifests are stale."); } @@ -176,11 +213,13 @@ export function main(args = process.argv.slice(2)) { fixtureFingerprint: source.fixtureFingerprint, manifestFingerprint: source.manifestFingerprint, variant: source.variant, baseSha: source.baseSha, archiveSha: source.archiveSha, sourceErrors: source.sourceErrors, immutable: source.immutable, layering: source.layering, + sourceMetadata: source.sourceMetadata, sourceMetadataErrors: source.sourceMetadataErrors, + sourceMetadataFingerprint: source.sourceMetadataFingerprint, measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1, - setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, evidence: [] }, gates: [], passed: false }; + setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, runnerdExitCode: null, evidence: [] }, gates: [], passed: false }; if (!source.sha || source.sourceErrors.length || !source.layering || !source.immutable) { writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); - console.error("Native completion source admission failed; commit the exact native-only source and fixtures before qualification."); + console.error(`Native completion source admission failed before providers: ${[...source.sourceErrors, ...source.sourceMetadataErrors].join("; ")}`); process.exitCode = 1; return report; } const sdk = runCommand(process.execPath, [join(root, "scripts/ensure-plugin-build-deps.mjs")], env, 5 * 60_000); @@ -188,9 +227,22 @@ export function main(args = process.argv.slice(2)) { const runner = sdk.status === 0 ? runCommand("pnpm", ["--filter", "@paperclipai/paperclip-runner", "build:typescript"], env) : null; report.setup.runnerTypeScriptExitCode = runner?.status ?? null; report.setup.evidence.push(capture(output, "setup-runner-typescript", runner)); - report.setup.passed = sdk.status === 0 && runner?.status === 0; + const hosted = env.GITHUB_ACTIONS === "true"; + const runnerd = sdk.status === 0 && runner?.status === 0 && !hosted ? runCommand("cargo", + ["build", "--locked", "--offline", "--workspace", "--bins"], env, + 10 * 60_000, join(root, "packages/paperclip-runner/runner")) : null; + report.setup.runnerdExitCode = runnerd?.status ?? null; + report.setup.runnerdProvenance = runnerdProvenance(source, env); + report.setup.evidence.push(capture(output, "setup-runnerd", hosted + ? { stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust build and unit calibration not executed in this hosted cell; reusing trusted same-run build." } + : runnerd)); + report.setup.passed = sdk.status === 0 && runner?.status === 0 && (hosted || runnerd?.status === 0) && report.setup.runnerdProvenance.passed; if (report.setup.passed) { report.setup.buildOutputFingerprint = buildOutputFingerprint(); + report.setup.runnerdSha256 = report.setup.runnerdProvenance.binarySha256; + report.setup.runnerdSelectedPath = report.setup.runnerdProvenance.selectedPath; + report.setup.runnerdSourceSha = source.sha; + report.setup.runnerdSourceFingerprint = source.fingerprint; for (const gate of nativeCompletionGates(source.variant)) { console.log(`Checking ${gate.id}: ${gate.name}`); const file = `${gate.id}.json`, run = runCommand(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files, @@ -202,25 +254,35 @@ export function main(args = process.argv.slice(2)) { evidence: existsSync(join(output, file)) ? { file, sha256: nativeSourceSha256(readFileSync(join(output, file))) } : null }); } const commands = [ - { id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"] }, + { id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs"] }, { id: "NC-typecheck", command: process.execPath, args: ["node_modules/typescript/bin/tsc", "-p", "tests/runner-e2e/tsconfig.json"] }, + ...(!hosted ? [{ id: "NC-rust-carrier", command: "cargo", args: ["test", "--locked", "--offline", + "-p", "paperclip-runner-core", "--lib", "provider_events::tests::preserves_closed_compatibility_terminal_tool_identity", "--", "--exact"], + cwd: join(root, "packages/paperclip-runner/runner") }] : []), { id: "NC-discovery", command: process.execPath, args: ["--import", "./server/node_modules/tsx/dist/loader.mjs", "tests/runner-e2e/launch.ts", "--list", "--suite", "native-completion"] }, ]; for (const command of commands) { console.log(`Checking ${command.id}`); - const run = runCommand(command.command, command.args, env); + const run = runCommand(command.command, command.args, env, 10 * 60_000, command.cwd ?? root); report.gates.push({ id: command.id, passed: run.status === 0 && (command.id !== "NC-discovery" || gradeNativeCompletionDiscovery(run.stdout, run.status).passed) && + (command.id !== "NC-rust-carrier" || gradeNativeCompletionRustCarrier(`${run.stdout ?? ""}\n${run.stderr ?? ""}`, run.status)) && (command.id !== "NC-node" || /# fail 0\b/.test(run.stdout) && /# tests [1-9]\d*\b/.test(run.stdout)), exitCode: run.status, evidence: capture(output, command.id, run) }); } + if (hosted) report.gates.push({ id: "NC-hosted-runnerd", name: "Trusted hosted runnerd provenance; Rust calibration not executed", + passed: report.setup.runnerdProvenance.passed, exitCode: 0, executed: false, calibration: "not_executed", + reuse: "trusted_same_run_build", total: 0, passedTests: 0, evidence: capture(output, "NC-hosted-runnerd", { + stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust unit calibration must be qualified by the separate exact-source local receipt and normal CI." }) }); const manifest = manifestCommands(env), combined = { stdout: manifest.map(run => run.stdout ?? "").join("\n"), stderr: manifest.map(run => run.stderr ?? "").join("\n") }; report.gates.push({ id: "NC-manifest", passed: manifest.every(run => run.status === 0), exitCode: manifest.find(run => run.status !== 0)?.status ?? 0, evidence: capture(output, "NC-manifest", combined) }); const after = currentSource(); report.passed = report.gates.every(gate => gate.passed) && after.immutable && after.layering && after.sha === source.sha && after.fingerprint === source.fingerprint && after.sourceErrors.length === 0 && - buildOutputFingerprint() === report.setup.buildOutputFingerprint; + after.sourceMetadataFingerprint === source.sourceMetadataFingerprint && after.sourceMetadataErrors.length === 0 && + buildOutputFingerprint() === report.setup.buildOutputFingerprint && + JSON.stringify(runnerdProvenance(after, env)) === JSON.stringify(report.setup.runnerdProvenance); } writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); console.log(`Native completion prerequisite evidence: ${join(output, "preflight.json")}`); diff --git a/tests/runner-e2e/native-completion-checks.test.mjs b/tests/runner-e2e/native-completion-checks.test.mjs index 77d684c7b4..07d15db189 100644 --- a/tests/runner-e2e/native-completion-checks.test.mjs +++ b/tests/runner-e2e/native-completion-checks.test.mjs @@ -4,9 +4,9 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { - nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery, + nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery, gradeNativeCompletionRustCarrier, assertNativeCompletionPreflightReceipt, assertRetainedNativeCompletionEvidence, nativeCompletionPrerequisiteEnvironment, - NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS, + NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS, nativeCompletionCommandGateIds, } from "./native-completion-checks.mjs"; import { nativeSourceSha256, NATIVE_COMPLETION_SOURCE_CONTRACT } from "./native-completion-source-contract.mjs"; @@ -65,15 +65,23 @@ test("native completion prerequisite excludes future credentials and auth overri OPENAI_API_KEY: "secret", ANTHROPIC_API_KEY: "secret", FUTURE_TOKEN: "secret", NODE_OPTIONS: "secret", GH_TOKEN: "secret" }), { PATH: "/bin", HOME: "/fixture", CI: "true" }); }); -const current = variant => ({ sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64), - manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), variant, immutable: true, layering: true, sourceErrors: [] }); +const runnerdProof = () => ({ schema: "paperclip.native-completion-runnerd-provenance.v1", mode: "fresh_local_build", passed: true, + selectedPath: "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", binarySha256: "9".repeat(64), + sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), archiveSha256: null, workflowRunId: null, + workflowRunAttempt: null, artifactName: null, errors: [] }); +const current = variant => ({ runnerdProvenance: runnerdProof(), sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64), + manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), runnerdSha256: "9".repeat(64), variant, immutable: true, layering: true, sourceErrors: [], sourceMetadataFingerprint: "f".repeat(64), sourceMetadataErrors: [] }); const receipt = variant => ({ schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, passed: true, providerCalls: 0, live: "not_run", sourceSha: current(variant).sha, sourceFingerprint: current(variant).fingerprint, fixtureFingerprint: current(variant).fixtureFingerprint, manifestFingerprint: current(variant).manifestFingerprint, variant, baseSha: NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha, archiveSha: NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha, immutable: true, layering: true, sourceErrors: [], + sourceMetadataFingerprint: current(variant).sourceMetadataFingerprint, sourceMetadataErrors: [], expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1, setup: { passed: true, sdkExitCode: 0, runnerTypeScriptExitCode: 0, buildOutputFingerprint: current(variant).buildOutputFingerprint, - evidence: ["setup-sdk.txt", "setup-runner-typescript.txt"].map(file => ({ file, sha256: "f".repeat(64) })) }, + runnerdExitCode: 0, runnerdSha256: current(variant).runnerdSha256, runnerdProvenance: runnerdProof(), + runnerdSelectedPath: runnerdProof().selectedPath, + runnerdSourceSha: current(variant).sha, runnerdSourceFingerprint: current(variant).fingerprint, + evidence: ["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].map(file => ({ file, sha256: "f".repeat(64) })) }, gates: [...nativeCompletionGates(variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS].map(id => ({ id, passed: true, exitCode: 0 })) }); for (const variant of ["candidate", "historical"]) test(`native completion prerequisite admits only the exact immutable ${variant} receipt`, () => { assert.equal(assertNativeCompletionPreflightReceipt(receipt(variant), current(variant)).passed, true); @@ -91,6 +99,15 @@ const negatives = [ ["changed build output", r => { r.setup.buildOutputFingerprint = "f".repeat(64); }], ["extra cells", r => { r.expectedCells = 7; }], ["extra turns", r => { r.expectedTurns = 7; }], ["additional attempts", r => { r.maximumAttemptsPerCell = 2; }], + ["missing runnerd provenance", r => { delete r.setup.runnerdProvenance; }], + ["wrong selected runnerd path", r => { r.setup.runnerdSelectedPath = "packages/paperclip-runner/dist/bin/paperclip-runnerd"; }], + ["claimed hosted reuse of local binary", r => { r.setup.runnerdProvenance.mode = "trusted_hosted_archive"; r.setup.runnerdExitCode = null; }], + ["failed runnerd build", r => { r.setup.runnerdExitCode = 1; }], + ["stale runnerd binary", r => { r.setup.runnerdSha256 = "8".repeat(64); }], + ["wrong runnerd source SHA", r => { r.setup.runnerdSourceSha = "b".repeat(40); }], + ["wrong runnerd source fingerprint", r => { r.setup.runnerdSourceFingerprint = "8".repeat(64); }], + ["changed hosted source metadata", r => { r.sourceMetadataFingerprint = "a".repeat(64); }], + ["source metadata errors", r => { r.sourceMetadataErrors.push("unknown lineage"); }], ["missing setup evidence", r => { r.setup.evidence.pop(); }], ["duplicate setup evidence", r => { r.setup.evidence[1] = r.setup.evidence[0]; }], ]; @@ -116,3 +133,40 @@ test("native completion prerequisite rejects changed or missing retained evidenc assert.throws(() => assertRetainedNativeCompletionEvidence(output, null)); } finally { rmSync(output, { recursive: true, force: true }); } }); + +test("native completion prerequisite requires the executed exact Rust carrier calibration", () => { + const output = "test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;"; + assert.equal(gradeNativeCompletionRustCarrier(output, 0), true); + assert.equal(gradeNativeCompletionRustCarrier(output, 1), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace(" ... ok", " ... FAILED"), 0), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace("1 passed", "0 passed"), 0), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace("preserves_closed_compatibility_terminal_tool_identity", "other_test"), 0), false); +}); + +function hostedFixture() { + const c = current("candidate"), r = receipt("candidate"); + c.runnerdProvenance = { ...runnerdProof(), mode: "trusted_hosted_archive", archiveSha256: "7".repeat(64), + workflowRunId: "12345", workflowRunAttempt: "2", artifactName: `runner-e2e-build-${c.sha}-12345-2` }; + r.setup.runnerdProvenance = structuredClone(c.runnerdProvenance); r.setup.runnerdExitCode = null; + r.gates = r.gates.map(gate => gate.id === "NC-rust-carrier" ? { ...gate, id: "NC-hosted-runnerd", executed: false, + calibration: "not_executed", total: 0, passedTests: 0, reuse: "trusted_same_run_build" } : gate); + return { c, r }; +} +test("native hosted receipt explicitly reuses verified runnerd bytes without claiming Rust calibration execution", () => { + const {c, r} = hostedFixture(); assert.equal(assertNativeCompletionPreflightReceipt(r, c).passed, true); + assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-rust-carrier"), false); + assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-hosted-runnerd"), true); +}); +for (const [name, mutate] of [ + ["executed Rust calibration", r => { r.gates.at(-1).executed = true; }], + ["positive Rust test count", r => { r.gates.at(-1).passedTests = 1; }], + ["missing explicit not-executed status", r => { delete r.gates.at(-1).calibration; }], + ["fresh Rust build claim", r => { r.setup.runnerdExitCode = 0; }], + ["different archive bytes", r => { r.setup.runnerdProvenance.archiveSha256 = "8".repeat(64); }], + ["different workflow run", r => { r.setup.runnerdProvenance.workflowRunId = "999"; }], +]) test(`native hosted receipt rejects ${name}`, () => { + const {c,r} = hostedFixture(); + // The hosted provenance gate is last in this deterministic fixture. + r.gates.sort((a,b) => a.id === "NC-hosted-runnerd" ? 1 : b.id === "NC-hosted-runnerd" ? -1 : 0); + mutate(r); assert.throws(() => assertNativeCompletionPreflightReceipt(r,c)); +}); diff --git a/tests/runner-e2e/native-completion-git-source.mjs b/tests/runner-e2e/native-completion-git-source.mjs new file mode 100644 index 0000000000..0da48024d4 --- /dev/null +++ b/tests/runner-e2e/native-completion-git-source.mjs @@ -0,0 +1,166 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { closeSync, constants, fstatSync, lstatSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, posix } from "node:path"; + +const hydrationDirectory = "runner-e2e-build"; +const archiveName = "runner-e2e-build-bundle.tar.gz"; +const checksumName = `${archiveName}.sha256`; +const hydrationPaths = [archiveName, checksumName].map(file => `${hydrationDirectory}/${file}`); +const sha256 = bytes => createHash("sha256").update(bytes).digest("hex"); + +/** A shallow commit still carries its hash-bound immediate parent in raw bytes. */ +export function verifyNativeCompletionParentAnchor({ sha, rawCommit, anchor }) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !/^[a-f0-9]{40}$/.test(anchor ?? "")) return false; + const bytes = Buffer.isBuffer(rawCommit) ? rawCommit : Buffer.from(rawCommit ?? ""); + const observed = createHash("sha1").update(`commit ${bytes.length}\0`).update(bytes).digest("hex"); + const parents = bytes.toString("utf8").split("\n\n", 1)[0].split("\n").filter(line => line.startsWith("parent ")); + return observed === sha && parents.length === 1 && parents[0] === `parent ${anchor}`; +} +function regularFile(path) { + const stat = lstatSync(path); + if (!stat.isFile() || stat.nlink !== 1) throw new Error("entry is not a regular, independently downloaded file"); + return stat; +} +function archiveDigest(path) { + const before = regularFile(path), fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + const opened = fstatSync(fd); + if (!opened.isFile() || opened.nlink !== 1 || opened.ino !== before.ino || opened.dev !== before.dev) + throw new Error("archive changed during verification"); + const hash = createHash("sha256"), buffer = Buffer.alloc(1024 * 1024); + let count; + while ((count = readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count)); + const after = fstatSync(fd); + if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) throw new Error("archive changed during verification"); + return hash.digest("hex"); + } finally { closeSync(fd); } +} +export function inspectNativeCompletionBuildHydration(repositoryRoot) { + const directory = join(repositoryRoot, hydrationDirectory), errors = []; + let stat; + try { stat = lstatSync(directory); } + catch (error) { + if (error.code === "ENOENT") return { present: false, verified: false, archiveSha256: null, checksumSha256: null, errors }; + return { present: true, verified: false, archiveSha256: null, checksumSha256: null, errors: ["build hydration directory is unreadable"] }; + } + let archiveSha256 = null, checksumSha256 = null; + try { + if (!stat.isDirectory()) throw new Error("directory must be a real directory, not a symlink or file"); + const entries = readdirSync(directory).sort(); + if (JSON.stringify(entries) !== JSON.stringify([archiveName, checksumName].sort())) + throw new Error("directory must contain exactly the archive and its checksum, with no extra entries"); + const checksumPath = join(directory, checksumName), checksumStat = regularFile(checksumPath); + if (checksumStat.size > 128) throw new Error("checksum record is malformed"); + const checksum = readFileSync(checksumPath), match = /^([a-f0-9]{64}) {2}runner-e2e-build-bundle\.tar\.gz\n?$/.exec(checksum.toString("utf8")); + if (!match) throw new Error("checksum must be one exact SHA256 record for the downloaded archive"); + checksumSha256 = sha256(checksum); archiveSha256 = archiveDigest(join(directory, archiveName)); + if (archiveSha256 !== match[1]) throw new Error("archive checksum verification failed"); + } catch (error) { errors.push(`build hydration ${error.message}`); } + return { present: true, verified: errors.length === 0, archiveSha256, checksumSha256, errors }; +} + +export const NATIVE_COMPLETION_RUNNERD_PATH = "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; +/** Mirror the production default selector, and reject staged binaries that would take precedence. */ +export function inspectNativeCompletionRunnerd({ repositoryRoot, sourceSha, sourceFingerprint, environment }) { + const errors = [], hosted = environment.GITHUB_ACTIONS === "true"; + const selectedPath = `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`; + const staged = join(repositoryRoot, `packages/paperclip-runner/dist/bin/paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`); + let binarySha256 = null, binaryBytes = null, archiveSha256 = null, archiveMemberBytes = null; + try { + try { lstatSync(staged); throw new Error("staged dist/bin runnerd would override the admitted debug executable"); } + catch (error) { if (error.code !== "ENOENT") throw error; } + let directory = repositoryRoot; + for (const part of selectedPath.split("/").slice(0, -1)) { + directory = join(directory, part); + if (!lstatSync(directory).isDirectory()) throw new Error("selected runnerd ancestor must be a real directory"); + } + const binary = join(repositoryRoot, selectedPath), stat = regularFile(binary); + if (process.platform !== "win32" && !(stat.mode & 0o111)) throw new Error("selected runnerd is not executable"); + binaryBytes = stat.size; + binarySha256 = archiveDigest(binary); + if (hosted) { + if (environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== sourceSha || !/^[a-f0-9]{40}$/.test(sourceSha ?? "")) + throw new Error("trusted hosted target SHA does not match admitted Git HEAD"); + if (![environment.GITHUB_RUN_ID, environment.GITHUB_RUN_ATTEMPT].every(value => typeof value === "string" && value === value.trim() && /^[1-9]\d*$/.test(value))) + throw new Error("trusted hosted workflow run and attempt are unavailable"); + const hydration = inspectNativeCompletionBuildHydration(repositoryRoot); + if (!hydration.verified) throw new Error(`hosted build archive is unverified: ${hydration.errors.join("; ")}`); + archiveSha256 = hydration.archiveSha256; + const archive = join(repositoryRoot, hydrationDirectory, archiveName); + const tar = args => spawnSync("tar", args, { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }); + const names = tar(["-tzf", archive]); + if (names.status !== 0) throw new Error("hosted build archive cannot be listed"); + const matching = names.stdout.split(/\r?\n/).filter(name => name && posix.normalize(name.replace(/^\/+/, "")) === selectedPath); + if (matching.length !== 1 || matching[0] !== selectedPath) + throw new Error("hosted archive must contain exactly one canonical selected runnerd member"); + const details = tar(["-tvzf", archive, selectedPath]); + const rows = details.stdout.trim().split(/\r?\n/); + if (details.status !== 0 || rows.length !== 1 || !rows[0].startsWith("-") || !rows[0].endsWith(` ${selectedPath}`)) + throw new Error("hosted selected runnerd member must be a regular file"); + // Keep daemon bytes out of JavaScript memory; the selected executable supplies the exact expected size. + const temporary = mkdtempSync(join(tmpdir(), "native-completion-runnerd-member-")), member = join(temporary, "runnerd"); + let fd; + try { + fd = openSync(member, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW ?? 0), 0o600); + const extracted = spawnSync("tar", ["-xOzf", archive, selectedPath], { timeout: 120_000, stdio: ["ignore", fd, "pipe"] }); + archiveMemberBytes = fstatSync(fd).size; + closeSync(fd); fd = undefined; + if (extracted.status !== 0 || archiveMemberBytes !== binaryBytes || archiveDigest(member) !== binarySha256) + throw new Error("selected runnerd byte count or hash differs from the verified hosted archive member"); + } finally { + if (fd !== undefined) closeSync(fd); + rmSync(temporary, { recursive: true, force: true }); + } + if (archiveDigest(archive) !== archiveSha256) throw new Error("hosted build archive changed during runnerd verification"); + } + } catch (error) { errors.push(`runnerd provenance ${error.message}`); } + return { schema: "paperclip.native-completion-runnerd-provenance.v1", mode: hosted ? "trusted_hosted_archive" : "fresh_local_build", + passed: errors.length === 0, selectedPath, binarySha256, binaryBytes, archiveMemberBytes, sourceSha, sourceFingerprint, archiveSha256, + workflowRunId: hosted ? environment.GITHUB_RUN_ID ?? null : null, + workflowRunAttempt: hosted ? environment.GITHUB_RUN_ATTEMPT ?? null : null, + artifactName: hosted ? `runner-e2e-build-${sourceSha}-${environment.GITHUB_RUN_ID}-${environment.GITHUB_RUN_ATTEMPT}` : null, + errors }; +} + +/** Preserve tracked-source cleanliness; allow only the verified workflow download. */ +export function inspectNativeCompletionSourceMetadata({ repositoryRoot, sourceFiles, baseSha, variant, shallowParentAnchors }) { + const git = (args, raw = false) => spawnSync("git", ["--no-replace-objects", ...args], { + cwd: repositoryRoot, encoding: raw ? undefined : "utf8", timeout: 30_000, + }); + const head = git(["rev-parse", "--verify", "HEAD"]), sha = head.status === 0 ? head.stdout.trim() : null; + const ancestor = git(["merge-base", "--is-ancestor", baseSha, "HEAD"]); + const shallow = git(["rev-parse", "--is-shallow-repository"]), isShallow = shallow.status === 0 && shallow.stdout.trim() === "true"; + const errors = []; + if (!/^[a-f0-9]{40}$/.test(sha ?? "")) errors.push("Git HEAD is unavailable or is not an immutable SHA1 commit"); + let layering = ancestor.status === 0, strategy = layering ? "full_ancestry" : null; + const anchor = shallowParentAnchors?.[variant] ?? null; + if (!layering && isShallow && anchor && sha) { + const commit = git(["cat-file", "commit", "HEAD"], true); + layering = commit.status === 0 && verifyNativeCompletionParentAnchor({ sha, rawCommit: commit.stdout, anchor }); + if (layering) strategy = "shallow_immediate_parent_anchor"; + } + if (!layering) errors.push(isShallow + ? "shallow Git HEAD does not have the sole hash-verified immediate parent admitted for this source variant" + : "full Git history does not prove the declared master base is an ancestor of HEAD"); + const tracked = git(["ls-files", "--error-unmatch", "--", ...sourceFiles]); + const clean = git(["diff", "--quiet", "HEAD", "--"]); + const status = git(["status", "--porcelain=v1", "-z", "--untracked-files=all"]); + const entries = status.status === 0 ? status.stdout.split("\0").filter(Boolean) : []; + const unexpected = entries.filter(entry => entry.slice(0, 3) !== "?? " || !hydrationPaths.includes(entry.slice(3))); + const hydration = inspectNativeCompletionBuildHydration(repositoryRoot); + if (tracked.status !== 0) errors.push("one or more admitted source paths are not tracked at HEAD"); + if (clean.status !== 0) errors.push("tracked source differs from HEAD"); + if (status.status !== 0) errors.push("Git worktree status is unavailable"); + if (unexpected.length) errors.push(`unexpected worktree entries: ${unexpected.join(", ")}`); + errors.push(...hydration.errors); + const immutable = tracked.status === 0 && clean.status === 0 && status.status === 0 && unexpected.length === 0 && + (!hydration.present || hydration.verified); + const metadata = { schema: "paperclip.native-completion-source-metadata.v1", + lineage: { strategy, shallow: isShallow, sha, baseSha, anchor: strategy === "shallow_immediate_parent_anchor" ? anchor : null }, + hydration: { present: hydration.present, verified: hydration.verified, archiveSha256: hydration.archiveSha256, checksumSha256: hydration.checksumSha256 }, + worktreeEntries: entries, errors }; + return { sha, layering, immutable, sourceMetadata: metadata, sourceMetadataErrors: errors, + sourceMetadataFingerprint: sha256(JSON.stringify(metadata)) }; +} diff --git a/tests/runner-e2e/native-completion-git-source.test.mjs b/tests/runner-e2e/native-completion-git-source.test.mjs new file mode 100644 index 0000000000..03681ee345 --- /dev/null +++ b/tests/runner-e2e/native-completion-git-source.test.mjs @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { after, test } from "node:test"; +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { inspectNativeCompletionSourceMetadata, inspectNativeCompletionBuildHydration, verifyNativeCompletionParentAnchor, + inspectNativeCompletionRunnerd, NATIVE_COMPLETION_RUNNERD_PATH } from "./native-completion-git-source.mjs"; + +const roots = []; +after(() => { for (const root of roots) rmSync(root, { recursive: true, force: true }); }); +const temporary = () => { const root = mkdtempSync(join(tmpdir(), "native-source-metadata-")); roots.push(root); return root; }; +const git = (root, args, input) => execFileSync("git", args, { cwd: root, input, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); +const digest = bytes => createHash("sha256").update(bytes).digest("hex"); +const archiveName = "runner-e2e-build-bundle.tar.gz", checksumName = `${archiveName}.sha256`; +function commit(root, tree, parents, message) { + const raw = `tree ${tree}\n${parents.map(parent => `parent ${parent}\n`).join("")}author Fixture 1 +0000\ncommitter Fixture 1 +0000\n\n${message}\n`; + return { raw, sha: git(root, ["hash-object", "-t", "commit", "-w", "--stdin"], raw) }; +} +function fixture(shallow = false, parentCount = 1) { + const source = temporary(); git(source, ["init", "--quiet"]); + writeFileSync(join(source, "source.txt"), "Immutable admitted source\n"); + const blob = git(source, ["hash-object", "-w", "source.txt"]); + const tree = git(source, ["mktree"], `100644 blob ${blob}\tsource.txt\n`); + const base = commit(source, tree, [], "Admitted base"), anchor = commit(source, tree, [base.sha], "Admitted native parent"); + const other = commit(source, tree, [base.sha], "Different parent"); + const head = commit(source, tree, parentCount === 2 ? [anchor.sha, other.sha] : [anchor.sha], "Native-only repair"); + git(source, ["update-ref", "refs/heads/fixture", head.sha]); git(source, ["checkout", "--quiet", "--force", "fixture"]); + let root = source; + if (shallow) { + root = join(temporary(), "checkout"); + git(source, ["clone", "--quiet", "--depth", "1", "--branch", "fixture", pathToFileURL(source).href, root]); + } + const input = { repositoryRoot: root, sourceFiles: ["source.txt"], baseSha: base.sha, variant: "candidate", + shallowParentAnchors: { candidate: anchor.sha, historical: other.sha } }; + return { root, source, base, anchor, other, head, input }; +} +function hydrate(root, bytes = Buffer.from("Exact downloaded build archive bytes")) { + const directory = join(root, "runner-e2e-build"); mkdirSync(directory); + writeFileSync(join(directory, archiveName), bytes); + writeFileSync(join(directory, checksumName), `${digest(bytes)} ${archiveName}\n`); + return directory; +} +test("native hosted-source metadata retains full ancestry and clean sources", () => { + const f = fixture(), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.layering, true); assert.equal(result.immutable, true); assert.deepEqual(result.sourceMetadataErrors, []); + assert.equal(result.sourceMetadata.lineage.strategy, "full_ancestry"); +}); +test("native hosted-source metadata accepts an actual shallow clone with the exact raw parent anchor", () => { + const f = fixture(true); hydrate(f.root); + assert.equal(git(f.root, ["rev-parse", "--is-shallow-repository"]), "true"); + assert.throws(() => git(f.root, ["merge-base", "--is-ancestor", f.base.sha, "HEAD"])); + const result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.sha, f.head.sha); assert.equal(result.layering, true); assert.equal(result.immutable, true); + assert.deepEqual(result.sourceMetadataErrors, []); + assert.equal(result.sourceMetadata.lineage.strategy, "shallow_immediate_parent_anchor"); + assert.equal(result.sourceMetadata.lineage.anchor, f.anchor.sha); + assert.equal(result.sourceMetadata.hydration.verified, true); +}); +test("native hosted-source metadata requires the variant-specific shallow anchor", () => { + const f = fixture(true), result = inspectNativeCompletionSourceMetadata({ ...f.input, variant: "historical" }); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified immediate parent"))); +}); +test("native hosted-source metadata never substitutes an anchor for missing full-clone ancestry", () => { + const f = fixture(), result = inspectNativeCompletionSourceMetadata({ ...f.input, baseSha: "f".repeat(40) }); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("full Git history"))); +}); +test("native hosted-source metadata rejects a shallow merge even with the admitted parent", () => { + const f = fixture(true, 2), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified"))); +}); +test("native hosted-source metadata cryptographically binds all raw HEAD bytes and the sole parent", () => { + const f = fixture(); + assert.equal(verifyNativeCompletionParentAnchor({ sha: f.head.sha, rawCommit: f.head.raw, anchor: f.anchor.sha }), true); + for (const input of [ + { sha: "f".repeat(40), rawCommit: f.head.raw, anchor: f.anchor.sha }, + { sha: f.head.sha, rawCommit: `${f.head.raw}altered`, anchor: f.anchor.sha }, + { sha: f.head.sha, rawCommit: f.head.raw, anchor: f.other.sha }, + { sha: f.base.sha, rawCommit: f.base.raw, anchor: f.anchor.sha }, + { sha: null, rawCommit: f.head.raw, anchor: f.anchor.sha }, + ]) assert.equal(verifyNativeCompletionParentAnchor(input), false); +}); +test("native hosted-source metadata rejects dirty tracked and untracked admitted source", () => { + const f = fixture(); hydrate(f.root); writeFileSync(join(f.root, "source.txt"), "Changed admitted source\n"); + let result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.includes("tracked source differs from HEAD")); + git(f.root, ["checkout", "HEAD", "--", "source.txt"]); writeFileSync(join(f.root, "extra-source.mjs"), "Unexpected code"); + result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("extra-source.mjs"))); + assert.equal(inspectNativeCompletionSourceMetadata({ ...f.input, sourceFiles: ["extra-source.mjs"] }).immutable, false); +}); +test("native hosted-source metadata verifies only the exact two downloaded regular files", () => { + const f = fixture(), directory = hydrate(f.root), first = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(first.immutable, true); assert.equal(first.sourceMetadata.hydration.verified, true); + assert.equal(inspectNativeCompletionSourceMetadata(f.input).sourceMetadataFingerprint, first.sourceMetadataFingerprint); + writeFileSync(join(directory, archiveName), "Different verified archive"); + writeFileSync(join(directory, checksumName), `${digest("Different verified archive")} ${archiveName}\n`); + const changed = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(changed.immutable, true); assert.notEqual(changed.sourceMetadataFingerprint, first.sourceMetadataFingerprint); +}); +const negatives = [ + ["missing checksum", (root, directory) => rmSync(join(directory, checksumName))], + ["checksum mismatch", (root, directory) => writeFileSync(join(directory, archiveName), "Unverified replacement")], + ["extra file", (root, directory) => writeFileSync(join(directory, "extra.mjs"), "Unexpected code")], + ["extra nested directory", (root, directory) => mkdirSync(join(directory, "nested"))], + ["malformed checksum", (root, directory) => writeFileSync(join(directory, checksumName), "malformed")], + ["wrong checksum filename", (root, directory) => writeFileSync(join(directory, checksumName), `${"a".repeat(64)} other.tar.gz\n`)], + ["multiple checksum records", (root, directory) => writeFileSync(join(directory, checksumName), readFileSync(join(directory, checksumName), "utf8").repeat(2))], + ["symlink archive", (root, directory) => { rmSync(join(directory, archiveName)); symlinkSync(join(root, "source.txt"), join(directory, archiveName)); }], + ["symlink checksum", (root, directory) => { rmSync(join(directory, checksumName)); symlinkSync(join(root, "source.txt"), join(directory, checksumName)); }], + ["symlink directory", (root, directory) => { rmSync(directory, { recursive: true }); symlinkSync(root, directory); }], +]; +for (const [name, mutate] of negatives) test(`native hosted-source metadata rejects ${name} hydration`, () => { + const f = fixture(), directory = hydrate(f.root); mutate(f.root, directory); + const hydration = inspectNativeCompletionBuildHydration(f.root), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(hydration.verified, false); assert.ok(hydration.errors.length > 0); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("build hydration"))); +}); +test("native hosted-source metadata rejects unrelated dirty trees despite verified hydration", () => { + const f = fixture(); hydrate(f.root); mkdirSync(join(f.root, "untrusted")); writeFileSync(join(f.root, "untrusted", "anything"), "extra"); + const result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("untrusted/anything"))); +}); + +function runnerdFixture() { + const root = temporary(), binary = join(root, NATIVE_COMPLETION_RUNNERD_PATH); + mkdirSync(join(binary, ".."), { recursive: true }); writeFileSync(binary, "Exact built runnerd fixture bytes"); chmodSync(binary, 0o755); + const directory = join(root, "runner-e2e-build"); mkdirSync(directory); + function pack(members = [NATIVE_COMPLETION_RUNNERD_PATH]) { + execFileSync("tar", ["-czf", join(directory, archiveName), ...members], { cwd: root }); + writeFileSync(join(directory, checksumName), `${digest(readFileSync(join(directory, archiveName)))} ${archiveName}\n`); + } + pack(); + const input = { repositoryRoot: root, sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), environment: { + GITHUB_ACTIONS: "true", PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "2" } }; + return { root, binary, directory, pack, input }; +} +test("native hosted runnerd proof binds the actual selected debug executable to the trusted same-run archive", () => { + const f = runnerdFixture(), proof = inspectNativeCompletionRunnerd(f.input); + assert.equal(proof.passed, true); assert.equal(proof.mode, "trusted_hosted_archive"); + assert.equal(proof.selectedPath, NATIVE_COMPLETION_RUNNERD_PATH); + assert.equal(proof.binarySha256, digest(readFileSync(f.binary))); + assert.equal(proof.binaryBytes, readFileSync(f.binary).length); assert.equal(proof.archiveMemberBytes, proof.binaryBytes); + assert.equal(proof.artifactName, `runner-e2e-build-${f.input.sourceSha}-12345-2`); + assert.deepEqual(proof.errors, []); + const local = inspectNativeCompletionRunnerd({ ...f.input, environment: {} }); + assert.equal(local.passed, true); assert.equal(local.mode, "fresh_local_build"); + assert.equal(local.archiveSha256, null); assert.equal(local.workflowRunId, null); + assert.equal(local.binaryBytes, readFileSync(f.binary).length); assert.equal(local.archiveMemberBytes, null); +}); +for (const [name, mutate] of [ + ["wrong trusted source SHA", f => { f.input.environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA = "c".repeat(40); }], + ["missing workflow run", f => { delete f.input.environment.GITHUB_RUN_ID; }], + ["malformed workflow attempt", f => { f.input.environment.GITHUB_RUN_ATTEMPT = "../2"; }], + ["newline in workflow run", f => { f.input.environment.GITHUB_RUN_ID = "12345\n"; }], + ["unverified archive", f => { writeFileSync(join(f.directory, archiveName), "Replacement"); }], + ["different selected executable", f => { writeFileSync(f.binary, "Stale binary"); }], + ["missing selected executable", f => { rmSync(f.binary); }], + ["nonexecutable selected binary", f => { chmodSync(f.binary, 0o644); }], + ["same-size different binary hash", f => { writeFileSync(f.binary, Buffer.alloc(readFileSync(f.binary).length, 0x58)); }], + ["symlink selected executable", f => { const bytes = readFileSync(f.binary); rmSync(f.binary); writeFileSync(join(f.root, "other"), bytes); symlinkSync(join(f.root, "other"), f.binary); }], + ["symlink executable ancestor", f => { const bytes = readFileSync(f.binary), directory = join(f.binary, ".."); rmSync(directory, { recursive: true }); mkdirSync(join(f.root, "other")); writeFileSync(join(f.root, "other", "paperclip-runnerd"), bytes); chmodSync(join(f.root, "other", "paperclip-runnerd"), 0o755); symlinkSync(join(f.root, "other"), directory); }], + ["staged executable override", f => { const staged = join(f.root, "packages/paperclip-runner/dist/bin"); mkdirSync(staged, { recursive: true }); writeFileSync(join(staged, "paperclip-runnerd"), "Unexpected staged binary"); }], + ["duplicate canonical archive members", f => { f.pack([NATIVE_COMPLETION_RUNNERD_PATH, NATIVE_COMPLETION_RUNNERD_PATH]); }], + ["noncanonical archive member", f => { f.pack([`./${NATIVE_COMPLETION_RUNNERD_PATH}`]); }], + ["missing archive member", f => { writeFileSync(join(f.root, "other"), "Other"); f.pack(["other"]); }], + ["symlink archive member", f => { rmSync(f.binary); writeFileSync(join(f.root, "other"), "Other"); symlinkSync(join(f.root, "other"), f.binary); f.pack(); rmSync(f.binary); writeFileSync(f.binary, "Other"); chmodSync(f.binary, 0o755); }], +]) test(`native hosted runnerd proof rejects ${name}`, () => { + const f = runnerdFixture(); mutate(f); + const proof = inspectNativeCompletionRunnerd(f.input); + assert.equal(proof.passed, false); assert.ok(proof.errors.length > 0); +}); diff --git a/tests/runner-e2e/native-completion-scoring.test.ts b/tests/runner-e2e/native-completion-scoring.test.ts index 6d06b50b65..8c476cbc32 100644 --- a/tests/runner-e2e/native-completion-scoring.test.ts +++ b/tests/runner-e2e/native-completion-scoring.test.ts @@ -1,4 +1,6 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; +import { rehydrateRunnerdItemNotification } from "../../packages/paperclip-runner/src/live/runnerd-codex-transport.js"; import { gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js"; type Row = Record; function sample(blocked = true, compatibility = false): NativeCompletionObservation { @@ -14,7 +16,7 @@ function sample(blocked = true, compatibility = false): NativeCompletionObservat runs: [{ id: "run", nativeIssueId: "issue", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", resultJson: { nativeResult: result } }], comments: [{ createdByRunId: "run", authorAgentId: "agent", body }], initial: { issueIds: [], agentIds: ["agent"] }, state: { issueIds: ["issue"], agentIds: ["agent"], documentCount: blocked ? 0 : 1, interactionCount: 0 }, workspaceChanged: false, - events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool" } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result), + events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool", name: tool } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result), compatibility ? event(3, "item.completed", { kind: "tool_result", item: { type: "tool_result", status: "completed", id: "tool" } }) : event(3, "tool.execution.completed", { name: tool, status: "completed", executionId: "tool" }), event(4, "item.completed", { kind: "agentMessage", channel: "final", item: { type: "agentMessage", phase: "final_answer", text: body, channel: "final" } }), event(5, "run.result.accepted", { result }, "control_plane"), event(6, "run.terminal", { runTerminalState: "succeeded", turnTerminalState: "completed" }, "control_plane")], @@ -23,6 +25,22 @@ function sample(blocked = true, compatibility = false): NativeCompletionObservat function payload(input: NativeCompletionObservation, index: number): Row { return ((input.events[index]!.payload as Row).prpEvent as Row).payload as Row; } describe("native completion independent oracle", () => { it.each([[true, false], [true, true], [false, false], [false, true]])("accepts disposition %s compatibility %s with final before late control-plane acceptance", (blocked, compatibility) => expect(gradeNativeCompletion(sample(blocked, compatibility)).passed).toBe(true)); + it.each(["paperclip_finish", "paperclip_block"])("carries normalized %s identity through rehydration into the exact-call oracle", name => { + // The Rust normalization calibration asserts these exact fixture bytes. + const fixture = JSON.parse(readFileSync(new URL("./fixtures/native-completion/terminal-tool-carrier.json", import.meta.url), "utf8")) as { + cases: Array<{ name: string; normalizedPayload: Row }>; + }; + const normalized = fixture.cases.find(value => value.name === name)!.normalizedPayload; + const started = rehydrateRunnerdItemNotification(normalized, "thread", "turn"); + expect(started.item).toMatchObject({ id: "terminal-call", type: "tool_call", name }); + const completed = rehydrateRunnerdItemNotification({ provider: "codex", itemId: "terminal-call", kind: "tool_result", status: "completed", channel: "detail", text: null }, "thread", "turn"); + const value = sample(name === "paperclip_block", true); + payload(value, 0).item = started.item; + payload(value, 2).item = completed.item; + expect(gradeNativeCompletion(value).passed).toBe(true); + (payload(value, 2).item as Row).id = "different-call"; + expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false); + }); it("accepts authoritative acceptance before the provider final", () => { const value = sample(); const events = [...value.events]; @@ -40,6 +58,13 @@ describe("native completion independent oracle", () => { else (payload(value, 2).item as Row).id = "unmatched"; expect(gradeNativeCompletion(value).passed).toBe(false); }); + it.each(["missing-name", "unrelated-named-tool", "contradictory-result-name"])("rejects compatibility finishing identity %s despite a matching ID", name => { + const value = sample(true, true); + if (name === "missing-name") delete (payload(value, 0).item as Row).name; + if (name === "unrelated-named-tool") (payload(value, 0).item as Row).name = "write_document"; + if (name === "contradictory-result-name") (payload(value, 2).item as Row).name = "write_document"; + expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false); + }); it.each(["extra-run", "retry", "continuation", "wrong-account", "wrong-agent", "wrong-disposition", "punctuated-action", "wrong-scope", "missing-final", "summary-fallback", "pre-tool-final", "post-admission-call", "missing-acceptance", "runner-acceptance", "failed-terminal", "event-gap", "binding-mismatch", "extra-task", "extra-agent", "extra-document", "interaction", "changed-workspace", "process-call", "marker-only", "contradiction", "wrong-reply-run"])("rejects %s", name => { const value = sample(); const run = value.runs[0]!; if (name === "extra-run") value.runs = [run, { ...run, id: "extra" }]; diff --git a/tests/runner-e2e/native-completion-scoring.ts b/tests/runner-e2e/native-completion-scoring.ts index b774fd34e3..86740c4580 100644 --- a/tests/runner-e2e/native-completion-scoring.ts +++ b/tests/runner-e2e/native-completion-scoring.ts @@ -83,6 +83,8 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) { || event.eventType === "item.completed" && start.eventType === "item.started" && (started.kind === "tool_call" || startedItem.type === "tool_call") && typeof item.id === "string" && item.id.length > 0 && item.id === startedItem.id + && terminalName(startedItem.name, expectedTool) + && (item.name === undefined || item.name === null || terminalName(item.name, expectedTool)) ); }); }); @@ -128,6 +130,6 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) { check("no-deployment-or-file-work", !input.workspaceChanged && !forbidden, "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed."); } - return { schema: "paperclip.native-completion-observation.v1", passed: checks.every(value => value.passed), checks, + return { schema: "paperclip.native-completion-observation.v2", passed: checks.every(value => value.passed), checks, limitations: ["Exact provider feedback identity/consumption is not measured by the public sequence."] }; } diff --git a/tests/runner-e2e/native-completion-source-contract.mjs b/tests/runner-e2e/native-completion-source-contract.mjs index a1755f008e..2fe54ce1f5 100644 --- a/tests/runner-e2e/native-completion-source-contract.mjs +++ b/tests/runner-e2e/native-completion-source-contract.mjs @@ -7,6 +7,10 @@ export const NATIVE_COMPLETION_SOURCE_CONTRACT = { "schema": "paperclip.native-completion-source-contract.v1", "baseSha": "59c07ede72dc08b8aba149a01cc11e0b7a204621", "archiveSha": "9138f570c341c251a5727c32d6615ce238bc8e03", + "shallowParentAnchors": { + "candidate": "e18c2cf9e96a4d31acb6d03ce918dfe223107d3f", + "historical": "459455acb11a012a97ad1b4afb77a1dc024a88bc" + }, "variants": { "candidate": { "packages/paperclip-runner/src/contracts/completion-result.ts": "bd4bb79d2be5c4ee3765df67dfb4da98ef95bc532cac872be40cb41a6bff6cda", @@ -61,6 +65,9 @@ export const NATIVE_COMPLETION_FIXTURE_FILES = [ "tests/runner-e2e/native-completion-admission.test.ts", "tests/runner-e2e/native-completion-checks.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", + "tests/runner-e2e/native-completion-git-source.mjs", + "tests/runner-e2e/native-completion-git-source.test.mjs", + "tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json", "tests/runner-e2e/native-completion-source-contract.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/automatic-retry.ts", @@ -108,6 +115,9 @@ export const NATIVE_COMPLETION_SOURCE_FILES = [...new Set([ "packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts", "packages/adapters/codex-local/src/index.ts", "packages/adapters/claude-local/src/index.ts", "packages/paperclip-runner/package.json", "tsconfig.base.json", + "packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs", + "packages/paperclip-runner/runner/Cargo.toml", "packages/paperclip-runner/runner/Cargo.lock", + "packages/paperclip-runner/runner/crates/runner-core/Cargo.toml", ])].sort(); export const nativeSourceSha256 = bytes => createHash("sha256").update(bytes).digest("hex"); diff --git a/tests/runner-e2e/native-completion-source-contract.test.mjs b/tests/runner-e2e/native-completion-source-contract.test.mjs index 73dd9525f9..e82b7b7f96 100644 --- a/tests/runner-e2e/native-completion-source-contract.test.mjs +++ b/tests/runner-e2e/native-completion-source-contract.test.mjs @@ -50,6 +50,15 @@ test("native source contract rejects an ambiguous identical variant map", () => const f = fixture(); f.contract.variants.historical = f.contract.variants.candidate; assert.equal(nativeCompletionSourceFingerprint(f.read, f.contract).variant, null); }); +test("native source contract binds common Rust carrier source and the shared behavioral projection fixture", () => { + const f = fixture(), before = nativeCompletionSourceFingerprint(f.read, f.contract); + f.bytes.set("packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs", Buffer.from("Changed carrier")); + const carrier = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.equal(carrier.variant, "candidate"); assert.notEqual(carrier.fingerprint, before.fingerprint); + f.bytes.set("tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json", Buffer.from("Changed projection")); + const projection = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.notEqual(projection.fingerprint, carrier.fingerprint); assert.notEqual(projection.fixtureFingerprint, carrier.fixtureFingerprint); +}); test("native source contract binds exactly five production and six variant assertion files", () => { const files = Object.keys(original.variants.candidate); assert.equal(files.length, 11); assert.equal(files.filter(file => file.endsWith(".test.ts")).length, 6); @@ -57,5 +66,9 @@ test("native source contract binds exactly five production and six variant asser assert.ok(files.every(file => original.variants.candidate[file] !== original.variants.historical[file])); assert.equal(original.baseSha, "59c07ede72dc08b8aba149a01cc11e0b7a204621"); assert.equal(original.archiveSha, "9138f570c341c251a5727c32d6615ce238bc8e03"); + assert.deepEqual(original.shallowParentAnchors, { + candidate: "e18c2cf9e96a4d31acb6d03ce918dfe223107d3f", + historical: "459455acb11a012a97ad1b4afb77a1dc024a88bc", + }); assert.ok(!NATIVE_COMPLETION_SOURCE_FILES.some(file => file.includes("stock-harness") || file.endsWith("issue-documents.md"))); });