mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-06 21:05:21 +02:00
Apply identical strict native qualification to historical descriptions
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
459455acb1
commit
5cafc2cbc4
11 files changed
+651
-32
No files matched your search
@@ -928,6 +928,18 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
|
||||
},
|
||||
"text": provider_item.get("text").and_then(Value::as_str).map(|value| bounded_text(value, MAX_TEXT_CHARS)),
|
||||
});
|
||||
// Preserve only actual terminal invocation identity in the compatibility
|
||||
// projection. Arguments, arbitrary tool names and result bodies stay omitted.
|
||||
if item_type == "tool_call" {
|
||||
if let Some(name @ ("paperclip_finish" | "paperclip_block")) =
|
||||
provider_item.get("name").and_then(Value::as_str)
|
||||
{
|
||||
payload
|
||||
.as_object_mut()
|
||||
.expect("item payload is an object")
|
||||
.insert("item".to_owned(), json!({ "name": name }));
|
||||
}
|
||||
}
|
||||
if !provider_phase.is_empty() {
|
||||
payload
|
||||
.as_object_mut()
|
||||
@@ -1562,6 +1574,40 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_closed_compatibility_terminal_tool_identity() {
|
||||
let fixture: Value = serde_json::from_str(include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../../../tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json"
|
||||
)))
|
||||
.unwrap();
|
||||
for case in fixture["cases"].as_array().unwrap() {
|
||||
let events = normalize_codex_notification("item/started", &case["providerInput"]);
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].event_type, "item.started");
|
||||
assert_eq!(events[0].payload, case["normalizedPayload"]);
|
||||
assert!(!events[0].payload.to_string().contains("not-for-the-log"));
|
||||
}
|
||||
for (item_type, name) in [
|
||||
("tool_call", Some("write_document")),
|
||||
("tool_call", Some("mcp.paperclip_finish")),
|
||||
("tool_call", None),
|
||||
("agentMessage", Some("paperclip_finish")),
|
||||
("tool_result", Some("paperclip_block")),
|
||||
] {
|
||||
let events = normalize_codex_notification(
|
||||
"item/completed",
|
||||
&json!({ "item": {
|
||||
"id": "terminal-call", "type": item_type, "name": name,
|
||||
"status": "completed", "arguments": {"secret": "not-for-the-log"},
|
||||
"result": {"secret": "not-for-the-log"}
|
||||
}}),
|
||||
);
|
||||
assert_eq!(events[0].payload.get("item"), None);
|
||||
assert!(!events[0].payload.to_string().contains("not-for-the-log"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforces_the_declared_safe_path_contract() {
|
||||
for location in [
|
||||
|
||||
@@ -21,8 +21,8 @@ effort, credentials, skills, permissions and provider configuration are inherite
|
||||
|
||||
The native oracle requires exactly one succeeded native run attributed to this
|
||||
issue, bound complete public events, one runner semantic proposal and authoritative
|
||||
control-plane acceptance/terminal. A matched finishing tool start/result precedes
|
||||
the actual provider final; no new invocation follows the proposal. The final
|
||||
control-plane acceptance/terminal. A matched finishing tool start/result with the expected native terminal name
|
||||
precedes the actual provider final; no new invocation follows the proposal. The final
|
||||
must be persisted as the run's agent reply. Acceptance may precede or follow the
|
||||
final. This proves observable ordering and durable disposition; it does not prove
|
||||
the provider consumed feedback or establish general coding-quality equivalence.
|
||||
@@ -34,6 +34,11 @@ six cells per variant, twelve expected provider turns total, with the existing
|
||||
per-cell deadlines (skill: original deadline; blocker: five minutes).
|
||||
|
||||
Candidate and baseline use identical executable fixtures and public oracles.
|
||||
Both include the same closed OpenCode compatibility projection: only actual
|
||||
`tool_call` invocations named `paperclip_finish` or `paperclip_block` retain a
|
||||
nested name, paired to the result by the same call ID. Arguments, arbitrary
|
||||
tool names and results are not added. Old anonymous OpenCode streams cannot
|
||||
retroactively satisfy this stricter identity check.
|
||||
Only the five native description/fingerprint source files and their six
|
||||
variant-specific unit tests may differ. The historical variant restores those
|
||||
eleven files from master, including v13; candidate uses the archived native change
|
||||
@@ -52,7 +57,14 @@ pnpm test:e2e:runner -- --list --suite native-completion
|
||||
Admission requires a committed clean exact source, variant-appropriate schema,
|
||||
tool bridge/catalog and resume tests, independent oracle/default/retry calibrations,
|
||||
original context-integrity calibrations, typecheck, manifest checks and exact
|
||||
six-cell discovery. Generic stock-suite gates remain unchanged. Paid hosted runs
|
||||
six-cell discovery. The Rust carrier calibration and built runner binary digest
|
||||
are retained with source/build provenance. Generic stock-suite gates remain unchanged. Paid hosted runs
|
||||
must dispatch the trusted default-branch workflow to distinct immutable target
|
||||
branches using the exact six IDs, preserving authorization and secret boundaries.
|
||||
The initial twelve hosted cells stopped at source admission with zero provider
|
||||
calls. Their receipts and frozen refs remain separate from the recovery comparison.
|
||||
Hosted depth-one checkouts use only the declared variant-specific immediate
|
||||
parent proof when full ancestry is unavailable. The only admitted untracked
|
||||
hydration is the exact build archive/checksum pair after checksum verification;
|
||||
extra files, symlinks or source dirt remain failures.
|
||||
Live results are pending until retained cell evidence is published.
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion-terminal-carrier.fixture.v1",
|
||||
"cases": [
|
||||
{
|
||||
"name": "paperclip_finish",
|
||||
"providerInput": {
|
||||
"item": {
|
||||
"id": "terminal-call",
|
||||
"type": "tool_call",
|
||||
"name": "paperclip_finish",
|
||||
"status": "inProgress",
|
||||
"arguments": {
|
||||
"secret": "not-for-the-log"
|
||||
}
|
||||
}
|
||||
},
|
||||
"normalizedPayload": {
|
||||
"provider": "codex",
|
||||
"itemId": "terminal-call",
|
||||
"kind": "tool_call",
|
||||
"status": "running",
|
||||
"channel": "detail",
|
||||
"text": null,
|
||||
"item": {
|
||||
"name": "paperclip_finish"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "paperclip_block",
|
||||
"providerInput": {
|
||||
"item": {
|
||||
"id": "terminal-call",
|
||||
"type": "tool_call",
|
||||
"name": "paperclip_block",
|
||||
"status": "inProgress",
|
||||
"arguments": {
|
||||
"secret": "not-for-the-log"
|
||||
}
|
||||
}
|
||||
},
|
||||
"normalizedPayload": {
|
||||
"provider": "codex",
|
||||
"itemId": "terminal-call",
|
||||
"kind": "tool_call",
|
||||
"status": "running",
|
||||
"channel": "detail",
|
||||
"text": null,
|
||||
"item": {
|
||||
"name": "paperclip_block"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
nativeCompletionSourceFingerprint, nativeSourceSha256,
|
||||
} from "./native-completion-source-contract.mjs";
|
||||
|
||||
import { inspectNativeCompletionSourceMetadata, inspectNativeCompletionRunnerd, NATIVE_COMPLETION_RUNNERD_PATH } from "./native-completion-git-source.mjs";
|
||||
|
||||
const root = resolve(import.meta.dirname, "../..");
|
||||
export const NATIVE_COMPLETION_PREFLIGHT_SCHEMA = "paperclip.native-completion-preflight.v1";
|
||||
export const NATIVE_COMPLETION_CELL_IDS = ["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile =>
|
||||
@@ -43,7 +45,10 @@ export function nativeCompletionGates(variant) {
|
||||
"retains the first provider_variance failure without a second attempt"] },
|
||||
];
|
||||
}
|
||||
export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery"];
|
||||
export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery", "NC-rust-carrier"];
|
||||
export function nativeCompletionCommandGateIds(mode) {
|
||||
return NATIVE_COMPLETION_COMMAND_GATE_IDS.map(id => mode === "trusted_hosted_archive" && id === "NC-rust-carrier" ? "NC-hosted-runnerd" : id);
|
||||
}
|
||||
export function gradeNativeCompletionGate(gate, report, exitCode) {
|
||||
const assertions = (report?.testResults ?? []).flatMap(file => file.assertionResults ?? []);
|
||||
const requirements = gate.required.map(name => ({ name, passed: assertions.some(assertion =>
|
||||
@@ -66,19 +71,20 @@ export function gradeNativeCompletionDiscovery(output, exitCode) {
|
||||
rows.every(row => row.length === 6 && row[1] === "native-completion" && row[2] === "native" && row[4]?.trim() && row[5]?.trim()),
|
||||
executionIds: ids, expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 };
|
||||
}
|
||||
export function gradeNativeCompletionRustCarrier(output, exitCode) {
|
||||
return exitCode === 0 && /^test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity \.\.\. ok$/m.test(output) &&
|
||||
/test result: ok\. 1 passed; 0 failed;/.test(output);
|
||||
}
|
||||
export function nativeCompletionPrerequisiteEnvironment(source) {
|
||||
return Object.fromEntries(["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL",
|
||||
"CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]]));
|
||||
"CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "PAPERCLIP_RUNNER_E2E_SOURCE_SHA"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]]));
|
||||
}
|
||||
function git(args) { return spawnSync("git", args, { cwd: root, encoding: "utf8" }); }
|
||||
function currentSource() {
|
||||
const source = nativeCompletionSourceFingerprint();
|
||||
const head = git(["rev-parse", "HEAD"]), ancestor = git(["merge-base", "--is-ancestor", source.baseSha, "HEAD"]);
|
||||
const tracked = git(["ls-files", "--error-unmatch", "--", ...NATIVE_COMPLETION_SOURCE_FILES]);
|
||||
const clean = git(["diff", "--quiet", "HEAD"]);
|
||||
const status = git(["status", "--porcelain", "--untracked-files=normal"]);
|
||||
return { ...source, sha: head.status === 0 ? head.stdout.trim() : null,
|
||||
layering: ancestor.status === 0, immutable: tracked.status === 0 && clean.status === 0 && status.status === 0 && status.stdout.trim() === "" };
|
||||
return { ...source, ...inspectNativeCompletionSourceMetadata({ repositoryRoot: root,
|
||||
sourceFiles: NATIVE_COMPLETION_SOURCE_FILES, baseSha: source.baseSha, variant: source.variant,
|
||||
shallowParentAnchors: NATIVE_COMPLETION_SOURCE_CONTRACT.shallowParentAnchors,
|
||||
}) };
|
||||
}
|
||||
function buildOutputFingerprint() {
|
||||
const hash = createHash("sha256");
|
||||
@@ -94,11 +100,25 @@ function buildOutputFingerprint() {
|
||||
if (!existsSync(join(root, directory, "index.js"))) throw new Error(`Missing prerequisite build output: ${directory}`);
|
||||
visit(directory);
|
||||
}
|
||||
const daemon = runnerdBinary();
|
||||
const bytes = readFileSync(daemon);
|
||||
hash.update(JSON.stringify(["debug/paperclip-runnerd", bytes.length])).update(bytes);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
function runnerdBinary() {
|
||||
return join(root, `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`);
|
||||
}
|
||||
function runnerdProvenance(source, env) {
|
||||
return inspectNativeCompletionRunnerd({ repositoryRoot: root, sourceSha: source.sha,
|
||||
sourceFingerprint: source.fingerprint, environment: env });
|
||||
}
|
||||
export function assertNativeCompletionPreflightReceipt(report, current) {
|
||||
const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS];
|
||||
if (report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 ||
|
||||
const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...nativeCompletionCommandGateIds(current.runnerdProvenance?.mode)];
|
||||
const hostedGate = report?.gates?.find(gate => gate.id === "NC-hosted-runnerd");
|
||||
const truthfulHosted = current.runnerdProvenance?.mode !== "trusted_hosted_archive" ||
|
||||
hostedGate?.executed === false && hostedGate?.calibration === "not_executed" && hostedGate?.total === 0 &&
|
||||
hostedGate?.passedTests === 0 && hostedGate?.reuse === "trusted_same_run_build";
|
||||
if (!truthfulHosted || report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 ||
|
||||
report.live !== "not_run" || report.sourceSha !== current.sha || !/^[a-f0-9]{40}$/.test(current.sha ?? "") ||
|
||||
![current.fingerprint, current.fixtureFingerprint, current.manifestFingerprint, current.buildOutputFingerprint]
|
||||
.every(value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) ||
|
||||
@@ -106,11 +126,21 @@ export function assertNativeCompletionPreflightReceipt(report, current) {
|
||||
report.manifestFingerprint !== current.manifestFingerprint || report.variant !== current.variant ||
|
||||
report.baseSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha || report.archiveSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha ||
|
||||
report.layering !== true || current.layering !== true || report.immutable !== true || current.immutable !== true ||
|
||||
!/^[a-f0-9]{64}$/.test(current.sourceMetadataFingerprint ?? "") ||
|
||||
report.sourceMetadataFingerprint !== current.sourceMetadataFingerprint ||
|
||||
!Array.isArray(report.sourceMetadataErrors) || report.sourceMetadataErrors.length !== 0 || current.sourceMetadataErrors?.length !== 0 ||
|
||||
!Array.isArray(report.sourceErrors) || report.sourceErrors.length !== 0 || current.sourceErrors?.length !== 0 ||
|
||||
report.setup?.passed !== true || report.setup.sdkExitCode !== 0 || report.setup.runnerTypeScriptExitCode !== 0 ||
|
||||
report.setup.runnerdExitCode !== (current.runnerdProvenance?.mode === "trusted_hosted_archive" ? null : 0) ||
|
||||
current.runnerdProvenance?.passed !== true || !["trusted_hosted_archive", "fresh_local_build"].includes(current.runnerdProvenance?.mode) ||
|
||||
JSON.stringify(report.setup.runnerdProvenance) !== JSON.stringify(current.runnerdProvenance) ||
|
||||
report.setup.runnerdSelectedPath !== current.runnerdProvenance.selectedPath ||
|
||||
report.setup.runnerdSha256 !== current.runnerdProvenance.binarySha256 ||
|
||||
!/^[a-f0-9]{64}$/.test(current.runnerdProvenance.binarySha256 ?? "") ||
|
||||
report.setup.runnerdSourceSha !== current.sha || report.setup.runnerdSourceFingerprint !== current.fingerprint ||
|
||||
report.setup.buildOutputFingerprint !== current.buildOutputFingerprint ||
|
||||
!Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 2 ||
|
||||
["setup-sdk.txt", "setup-runner-typescript.txt"].some(file => report.setup.evidence.filter(row =>
|
||||
!Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 3 ||
|
||||
["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].some(file => report.setup.evidence.filter(row =>
|
||||
row.file === file && /^[a-f0-9]{64}$/.test(row.sha256 ?? "")).length !== 1) ||
|
||||
!Array.isArray(report.gates) || report.gates.length !== expected.length ||
|
||||
expected.some(id => report.gates.filter(gate => gate.id === id && gate.passed === true && gate.exitCode === 0).length !== 1) ||
|
||||
@@ -141,7 +171,8 @@ function manifestCommands(env) {
|
||||
export function main(args = process.argv.slice(2)) {
|
||||
if (args.includes("--list")) {
|
||||
console.log(JSON.stringify({ variants: ["candidate", "historical"], gates: nativeCompletionGates("candidate"),
|
||||
commands: NATIVE_COMPLETION_COMMAND_GATE_IDS, cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return;
|
||||
commands: { local: nativeCompletionCommandGateIds("fresh_local_build"), hosted: nativeCompletionCommandGateIds("trusted_hosted_archive") },
|
||||
cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return;
|
||||
}
|
||||
if (args.some(arg => !arg.startsWith("--output-dir=") && !arg.startsWith("--verify=")))
|
||||
throw new Error("Use --list, --output-dir=<path> or --verify=<receipt>; never paid providers.");
|
||||
@@ -149,7 +180,7 @@ export function main(args = process.argv.slice(2)) {
|
||||
const verify = args.find(arg => arg.startsWith("--verify="))?.slice("--verify=".length);
|
||||
if (verify) {
|
||||
const report = assertNativeCompletionPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), {
|
||||
...source, buildOutputFingerprint: buildOutputFingerprint(),
|
||||
...source, buildOutputFingerprint: buildOutputFingerprint(), runnerdProvenance: runnerdProvenance(source, env),
|
||||
});
|
||||
const output = resolve(verify, "..");
|
||||
for (const evidence of report.setup.evidence) assertRetainedNativeCompletionEvidence(output, evidence);
|
||||
@@ -158,12 +189,18 @@ export function main(args = process.argv.slice(2)) {
|
||||
const grade = gradeNativeCompletionGate(gate, JSON.parse(assertRetainedNativeCompletionEvidence(output, retained.evidence)), 0);
|
||||
if (!grade.passed) throw new Error(`Missing or failed retained native prerequisite assertions: ${gate.id}`);
|
||||
}
|
||||
for (const id of NATIVE_COMPLETION_COMMAND_GATE_IDS) {
|
||||
for (const id of nativeCompletionCommandGateIds(report.setup.runnerdProvenance.mode)) {
|
||||
const retained = report.gates.find(row => row.id === id), text = assertRetainedNativeCompletionEvidence(output, retained.evidence);
|
||||
if (id === "NC-node" && (!/# fail 0\b/.test(text) || !/# tests [1-9]\d*\b/.test(text)))
|
||||
throw new Error("Missing passing native admission calibrations.");
|
||||
if (id === "NC-discovery" && !gradeNativeCompletionDiscovery(text.split("\n\n")[0], 0).passed)
|
||||
throw new Error("Native completion six-cell discovery changed.");
|
||||
if (id === "NC-rust-carrier" && !gradeNativeCompletionRustCarrier(text, 0))
|
||||
throw new Error("Missing passing retained Rust terminal-tool carrier calibration.");
|
||||
if (id === "NC-hosted-runnerd" && (retained.executed !== false || retained.calibration !== "not_executed" ||
|
||||
retained.total !== 0 || retained.passedTests !== 0 ||
|
||||
JSON.stringify(JSON.parse(text.split("\n\n")[0])) !== JSON.stringify(report.setup.runnerdProvenance)))
|
||||
throw new Error("Hosted runnerd reuse must retain exact binary proof and report Rust calibration not executed.");
|
||||
if (id === "NC-manifest" && manifestCommands(env).some(run => run.status !== 0))
|
||||
throw new Error("Generated native capability manifests are stale.");
|
||||
}
|
||||
@@ -176,11 +213,13 @@ export function main(args = process.argv.slice(2)) {
|
||||
fixtureFingerprint: source.fixtureFingerprint, manifestFingerprint: source.manifestFingerprint,
|
||||
variant: source.variant, baseSha: source.baseSha, archiveSha: source.archiveSha,
|
||||
sourceErrors: source.sourceErrors, immutable: source.immutable, layering: source.layering,
|
||||
sourceMetadata: source.sourceMetadata, sourceMetadataErrors: source.sourceMetadataErrors,
|
||||
sourceMetadataFingerprint: source.sourceMetadataFingerprint,
|
||||
measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1,
|
||||
setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, evidence: [] }, gates: [], passed: false };
|
||||
setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, runnerdExitCode: null, evidence: [] }, gates: [], passed: false };
|
||||
if (!source.sha || source.sourceErrors.length || !source.layering || !source.immutable) {
|
||||
writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n");
|
||||
console.error("Native completion source admission failed; commit the exact native-only source and fixtures before qualification.");
|
||||
console.error(`Native completion source admission failed before providers: ${[...source.sourceErrors, ...source.sourceMetadataErrors].join("; ")}`);
|
||||
process.exitCode = 1; return report;
|
||||
}
|
||||
const sdk = runCommand(process.execPath, [join(root, "scripts/ensure-plugin-build-deps.mjs")], env, 5 * 60_000);
|
||||
@@ -188,9 +227,22 @@ export function main(args = process.argv.slice(2)) {
|
||||
const runner = sdk.status === 0 ? runCommand("pnpm", ["--filter", "@paperclipai/paperclip-runner", "build:typescript"], env) : null;
|
||||
report.setup.runnerTypeScriptExitCode = runner?.status ?? null;
|
||||
report.setup.evidence.push(capture(output, "setup-runner-typescript", runner));
|
||||
report.setup.passed = sdk.status === 0 && runner?.status === 0;
|
||||
const hosted = env.GITHUB_ACTIONS === "true";
|
||||
const runnerd = sdk.status === 0 && runner?.status === 0 && !hosted ? runCommand("cargo",
|
||||
["build", "--locked", "--offline", "--workspace", "--bins"], env,
|
||||
10 * 60_000, join(root, "packages/paperclip-runner/runner")) : null;
|
||||
report.setup.runnerdExitCode = runnerd?.status ?? null;
|
||||
report.setup.runnerdProvenance = runnerdProvenance(source, env);
|
||||
report.setup.evidence.push(capture(output, "setup-runnerd", hosted
|
||||
? { stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust build and unit calibration not executed in this hosted cell; reusing trusted same-run build." }
|
||||
: runnerd));
|
||||
report.setup.passed = sdk.status === 0 && runner?.status === 0 && (hosted || runnerd?.status === 0) && report.setup.runnerdProvenance.passed;
|
||||
if (report.setup.passed) {
|
||||
report.setup.buildOutputFingerprint = buildOutputFingerprint();
|
||||
report.setup.runnerdSha256 = report.setup.runnerdProvenance.binarySha256;
|
||||
report.setup.runnerdSelectedPath = report.setup.runnerdProvenance.selectedPath;
|
||||
report.setup.runnerdSourceSha = source.sha;
|
||||
report.setup.runnerdSourceFingerprint = source.fingerprint;
|
||||
for (const gate of nativeCompletionGates(source.variant)) {
|
||||
console.log(`Checking ${gate.id}: ${gate.name}`);
|
||||
const file = `${gate.id}.json`, run = runCommand(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files,
|
||||
@@ -202,25 +254,35 @@ export function main(args = process.argv.slice(2)) {
|
||||
evidence: existsSync(join(output, file)) ? { file, sha256: nativeSourceSha256(readFileSync(join(output, file))) } : null });
|
||||
}
|
||||
const commands = [
|
||||
{ id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"] },
|
||||
{ id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs"] },
|
||||
{ id: "NC-typecheck", command: process.execPath, args: ["node_modules/typescript/bin/tsc", "-p", "tests/runner-e2e/tsconfig.json"] },
|
||||
...(!hosted ? [{ id: "NC-rust-carrier", command: "cargo", args: ["test", "--locked", "--offline",
|
||||
"-p", "paperclip-runner-core", "--lib", "provider_events::tests::preserves_closed_compatibility_terminal_tool_identity", "--", "--exact"],
|
||||
cwd: join(root, "packages/paperclip-runner/runner") }] : []),
|
||||
{ id: "NC-discovery", command: process.execPath, args: ["--import", "./server/node_modules/tsx/dist/loader.mjs", "tests/runner-e2e/launch.ts", "--list", "--suite", "native-completion"] },
|
||||
];
|
||||
for (const command of commands) {
|
||||
console.log(`Checking ${command.id}`);
|
||||
const run = runCommand(command.command, command.args, env);
|
||||
const run = runCommand(command.command, command.args, env, 10 * 60_000, command.cwd ?? root);
|
||||
report.gates.push({ id: command.id, passed: run.status === 0 &&
|
||||
(command.id !== "NC-discovery" || gradeNativeCompletionDiscovery(run.stdout, run.status).passed) &&
|
||||
(command.id !== "NC-rust-carrier" || gradeNativeCompletionRustCarrier(`${run.stdout ?? ""}\n${run.stderr ?? ""}`, run.status)) &&
|
||||
(command.id !== "NC-node" || /# fail 0\b/.test(run.stdout) && /# tests [1-9]\d*\b/.test(run.stdout)),
|
||||
exitCode: run.status, evidence: capture(output, command.id, run) });
|
||||
}
|
||||
if (hosted) report.gates.push({ id: "NC-hosted-runnerd", name: "Trusted hosted runnerd provenance; Rust calibration not executed",
|
||||
passed: report.setup.runnerdProvenance.passed, exitCode: 0, executed: false, calibration: "not_executed",
|
||||
reuse: "trusted_same_run_build", total: 0, passedTests: 0, evidence: capture(output, "NC-hosted-runnerd", {
|
||||
stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust unit calibration must be qualified by the separate exact-source local receipt and normal CI." }) });
|
||||
const manifest = manifestCommands(env), combined = { stdout: manifest.map(run => run.stdout ?? "").join("\n"), stderr: manifest.map(run => run.stderr ?? "").join("\n") };
|
||||
report.gates.push({ id: "NC-manifest", passed: manifest.every(run => run.status === 0),
|
||||
exitCode: manifest.find(run => run.status !== 0)?.status ?? 0, evidence: capture(output, "NC-manifest", combined) });
|
||||
const after = currentSource();
|
||||
report.passed = report.gates.every(gate => gate.passed) && after.immutable && after.layering &&
|
||||
after.sha === source.sha && after.fingerprint === source.fingerprint && after.sourceErrors.length === 0 &&
|
||||
buildOutputFingerprint() === report.setup.buildOutputFingerprint;
|
||||
after.sourceMetadataFingerprint === source.sourceMetadataFingerprint && after.sourceMetadataErrors.length === 0 &&
|
||||
buildOutputFingerprint() === report.setup.buildOutputFingerprint &&
|
||||
JSON.stringify(runnerdProvenance(after, env)) === JSON.stringify(report.setup.runnerdProvenance);
|
||||
}
|
||||
writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n");
|
||||
console.log(`Native completion prerequisite evidence: ${join(output, "preflight.json")}`);
|
||||
|
||||
@@ -4,9 +4,9 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery,
|
||||
nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery, gradeNativeCompletionRustCarrier,
|
||||
assertNativeCompletionPreflightReceipt, assertRetainedNativeCompletionEvidence, nativeCompletionPrerequisiteEnvironment,
|
||||
NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS,
|
||||
NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS, nativeCompletionCommandGateIds,
|
||||
} from "./native-completion-checks.mjs";
|
||||
import { nativeSourceSha256, NATIVE_COMPLETION_SOURCE_CONTRACT } from "./native-completion-source-contract.mjs";
|
||||
|
||||
@@ -65,15 +65,23 @@ test("native completion prerequisite excludes future credentials and auth overri
|
||||
OPENAI_API_KEY: "secret", ANTHROPIC_API_KEY: "secret", FUTURE_TOKEN: "secret", NODE_OPTIONS: "secret", GH_TOKEN: "secret" }),
|
||||
{ PATH: "/bin", HOME: "/fixture", CI: "true" });
|
||||
});
|
||||
const current = variant => ({ sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64),
|
||||
manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), variant, immutable: true, layering: true, sourceErrors: [] });
|
||||
const runnerdProof = () => ({ schema: "paperclip.native-completion-runnerd-provenance.v1", mode: "fresh_local_build", passed: true,
|
||||
selectedPath: "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", binarySha256: "9".repeat(64),
|
||||
sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), archiveSha256: null, workflowRunId: null,
|
||||
workflowRunAttempt: null, artifactName: null, errors: [] });
|
||||
const current = variant => ({ runnerdProvenance: runnerdProof(), sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64),
|
||||
manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), runnerdSha256: "9".repeat(64), variant, immutable: true, layering: true, sourceErrors: [], sourceMetadataFingerprint: "f".repeat(64), sourceMetadataErrors: [] });
|
||||
const receipt = variant => ({ schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, passed: true, providerCalls: 0, live: "not_run",
|
||||
sourceSha: current(variant).sha, sourceFingerprint: current(variant).fingerprint, fixtureFingerprint: current(variant).fixtureFingerprint,
|
||||
manifestFingerprint: current(variant).manifestFingerprint, variant,
|
||||
baseSha: NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha, archiveSha: NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha, immutable: true, layering: true, sourceErrors: [],
|
||||
sourceMetadataFingerprint: current(variant).sourceMetadataFingerprint, sourceMetadataErrors: [],
|
||||
expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1,
|
||||
setup: { passed: true, sdkExitCode: 0, runnerTypeScriptExitCode: 0, buildOutputFingerprint: current(variant).buildOutputFingerprint,
|
||||
evidence: ["setup-sdk.txt", "setup-runner-typescript.txt"].map(file => ({ file, sha256: "f".repeat(64) })) },
|
||||
runnerdExitCode: 0, runnerdSha256: current(variant).runnerdSha256, runnerdProvenance: runnerdProof(),
|
||||
runnerdSelectedPath: runnerdProof().selectedPath,
|
||||
runnerdSourceSha: current(variant).sha, runnerdSourceFingerprint: current(variant).fingerprint,
|
||||
evidence: ["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].map(file => ({ file, sha256: "f".repeat(64) })) },
|
||||
gates: [...nativeCompletionGates(variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS].map(id => ({ id, passed: true, exitCode: 0 })) });
|
||||
for (const variant of ["candidate", "historical"]) test(`native completion prerequisite admits only the exact immutable ${variant} receipt`, () => {
|
||||
assert.equal(assertNativeCompletionPreflightReceipt(receipt(variant), current(variant)).passed, true);
|
||||
@@ -91,6 +99,15 @@ const negatives = [
|
||||
["changed build output", r => { r.setup.buildOutputFingerprint = "f".repeat(64); }],
|
||||
["extra cells", r => { r.expectedCells = 7; }], ["extra turns", r => { r.expectedTurns = 7; }],
|
||||
["additional attempts", r => { r.maximumAttemptsPerCell = 2; }],
|
||||
["missing runnerd provenance", r => { delete r.setup.runnerdProvenance; }],
|
||||
["wrong selected runnerd path", r => { r.setup.runnerdSelectedPath = "packages/paperclip-runner/dist/bin/paperclip-runnerd"; }],
|
||||
["claimed hosted reuse of local binary", r => { r.setup.runnerdProvenance.mode = "trusted_hosted_archive"; r.setup.runnerdExitCode = null; }],
|
||||
["failed runnerd build", r => { r.setup.runnerdExitCode = 1; }],
|
||||
["stale runnerd binary", r => { r.setup.runnerdSha256 = "8".repeat(64); }],
|
||||
["wrong runnerd source SHA", r => { r.setup.runnerdSourceSha = "b".repeat(40); }],
|
||||
["wrong runnerd source fingerprint", r => { r.setup.runnerdSourceFingerprint = "8".repeat(64); }],
|
||||
["changed hosted source metadata", r => { r.sourceMetadataFingerprint = "a".repeat(64); }],
|
||||
["source metadata errors", r => { r.sourceMetadataErrors.push("unknown lineage"); }],
|
||||
["missing setup evidence", r => { r.setup.evidence.pop(); }],
|
||||
["duplicate setup evidence", r => { r.setup.evidence[1] = r.setup.evidence[0]; }],
|
||||
];
|
||||
@@ -116,3 +133,40 @@ test("native completion prerequisite rejects changed or missing retained evidenc
|
||||
assert.throws(() => assertRetainedNativeCompletionEvidence(output, null));
|
||||
} finally { rmSync(output, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("native completion prerequisite requires the executed exact Rust carrier calibration", () => {
|
||||
const output = "test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;";
|
||||
assert.equal(gradeNativeCompletionRustCarrier(output, 0), true);
|
||||
assert.equal(gradeNativeCompletionRustCarrier(output, 1), false);
|
||||
assert.equal(gradeNativeCompletionRustCarrier(output.replace(" ... ok", " ... FAILED"), 0), false);
|
||||
assert.equal(gradeNativeCompletionRustCarrier(output.replace("1 passed", "0 passed"), 0), false);
|
||||
assert.equal(gradeNativeCompletionRustCarrier(output.replace("preserves_closed_compatibility_terminal_tool_identity", "other_test"), 0), false);
|
||||
});
|
||||
|
||||
function hostedFixture() {
|
||||
const c = current("candidate"), r = receipt("candidate");
|
||||
c.runnerdProvenance = { ...runnerdProof(), mode: "trusted_hosted_archive", archiveSha256: "7".repeat(64),
|
||||
workflowRunId: "12345", workflowRunAttempt: "2", artifactName: `runner-e2e-build-${c.sha}-12345-2` };
|
||||
r.setup.runnerdProvenance = structuredClone(c.runnerdProvenance); r.setup.runnerdExitCode = null;
|
||||
r.gates = r.gates.map(gate => gate.id === "NC-rust-carrier" ? { ...gate, id: "NC-hosted-runnerd", executed: false,
|
||||
calibration: "not_executed", total: 0, passedTests: 0, reuse: "trusted_same_run_build" } : gate);
|
||||
return { c, r };
|
||||
}
|
||||
test("native hosted receipt explicitly reuses verified runnerd bytes without claiming Rust calibration execution", () => {
|
||||
const {c, r} = hostedFixture(); assert.equal(assertNativeCompletionPreflightReceipt(r, c).passed, true);
|
||||
assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-rust-carrier"), false);
|
||||
assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-hosted-runnerd"), true);
|
||||
});
|
||||
for (const [name, mutate] of [
|
||||
["executed Rust calibration", r => { r.gates.at(-1).executed = true; }],
|
||||
["positive Rust test count", r => { r.gates.at(-1).passedTests = 1; }],
|
||||
["missing explicit not-executed status", r => { delete r.gates.at(-1).calibration; }],
|
||||
["fresh Rust build claim", r => { r.setup.runnerdExitCode = 0; }],
|
||||
["different archive bytes", r => { r.setup.runnerdProvenance.archiveSha256 = "8".repeat(64); }],
|
||||
["different workflow run", r => { r.setup.runnerdProvenance.workflowRunId = "999"; }],
|
||||
]) test(`native hosted receipt rejects ${name}`, () => {
|
||||
const {c,r} = hostedFixture();
|
||||
// The hosted provenance gate is last in this deterministic fixture.
|
||||
r.gates.sort((a,b) => a.id === "NC-hosted-runnerd" ? 1 : b.id === "NC-hosted-runnerd" ? -1 : 0);
|
||||
mutate(r); assert.throws(() => assertNativeCompletionPreflightReceipt(r,c));
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { closeSync, constants, fstatSync, lstatSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, posix } from "node:path";
|
||||
|
||||
const hydrationDirectory = "runner-e2e-build";
|
||||
const archiveName = "runner-e2e-build-bundle.tar.gz";
|
||||
const checksumName = `${archiveName}.sha256`;
|
||||
const hydrationPaths = [archiveName, checksumName].map(file => `${hydrationDirectory}/${file}`);
|
||||
const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
/** A shallow commit still carries its hash-bound immediate parent in raw bytes. */
|
||||
export function verifyNativeCompletionParentAnchor({ sha, rawCommit, anchor }) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !/^[a-f0-9]{40}$/.test(anchor ?? "")) return false;
|
||||
const bytes = Buffer.isBuffer(rawCommit) ? rawCommit : Buffer.from(rawCommit ?? "");
|
||||
const observed = createHash("sha1").update(`commit ${bytes.length}\0`).update(bytes).digest("hex");
|
||||
const parents = bytes.toString("utf8").split("\n\n", 1)[0].split("\n").filter(line => line.startsWith("parent "));
|
||||
return observed === sha && parents.length === 1 && parents[0] === `parent ${anchor}`;
|
||||
}
|
||||
function regularFile(path) {
|
||||
const stat = lstatSync(path);
|
||||
if (!stat.isFile() || stat.nlink !== 1) throw new Error("entry is not a regular, independently downloaded file");
|
||||
return stat;
|
||||
}
|
||||
function archiveDigest(path) {
|
||||
const before = regularFile(path), fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
||||
try {
|
||||
const opened = fstatSync(fd);
|
||||
if (!opened.isFile() || opened.nlink !== 1 || opened.ino !== before.ino || opened.dev !== before.dev)
|
||||
throw new Error("archive changed during verification");
|
||||
const hash = createHash("sha256"), buffer = Buffer.alloc(1024 * 1024);
|
||||
let count;
|
||||
while ((count = readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count));
|
||||
const after = fstatSync(fd);
|
||||
if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) throw new Error("archive changed during verification");
|
||||
return hash.digest("hex");
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
export function inspectNativeCompletionBuildHydration(repositoryRoot) {
|
||||
const directory = join(repositoryRoot, hydrationDirectory), errors = [];
|
||||
let stat;
|
||||
try { stat = lstatSync(directory); }
|
||||
catch (error) {
|
||||
if (error.code === "ENOENT") return { present: false, verified: false, archiveSha256: null, checksumSha256: null, errors };
|
||||
return { present: true, verified: false, archiveSha256: null, checksumSha256: null, errors: ["build hydration directory is unreadable"] };
|
||||
}
|
||||
let archiveSha256 = null, checksumSha256 = null;
|
||||
try {
|
||||
if (!stat.isDirectory()) throw new Error("directory must be a real directory, not a symlink or file");
|
||||
const entries = readdirSync(directory).sort();
|
||||
if (JSON.stringify(entries) !== JSON.stringify([archiveName, checksumName].sort()))
|
||||
throw new Error("directory must contain exactly the archive and its checksum, with no extra entries");
|
||||
const checksumPath = join(directory, checksumName), checksumStat = regularFile(checksumPath);
|
||||
if (checksumStat.size > 128) throw new Error("checksum record is malformed");
|
||||
const checksum = readFileSync(checksumPath), match = /^([a-f0-9]{64}) {2}runner-e2e-build-bundle\.tar\.gz\n?$/.exec(checksum.toString("utf8"));
|
||||
if (!match) throw new Error("checksum must be one exact SHA256 record for the downloaded archive");
|
||||
checksumSha256 = sha256(checksum); archiveSha256 = archiveDigest(join(directory, archiveName));
|
||||
if (archiveSha256 !== match[1]) throw new Error("archive checksum verification failed");
|
||||
} catch (error) { errors.push(`build hydration ${error.message}`); }
|
||||
return { present: true, verified: errors.length === 0, archiveSha256, checksumSha256, errors };
|
||||
}
|
||||
|
||||
export const NATIVE_COMPLETION_RUNNERD_PATH = "packages/paperclip-runner/runner/target/debug/paperclip-runnerd";
|
||||
/** Mirror the production default selector, and reject staged binaries that would take precedence. */
|
||||
export function inspectNativeCompletionRunnerd({ repositoryRoot, sourceSha, sourceFingerprint, environment }) {
|
||||
const errors = [], hosted = environment.GITHUB_ACTIONS === "true";
|
||||
const selectedPath = `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`;
|
||||
const staged = join(repositoryRoot, `packages/paperclip-runner/dist/bin/paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`);
|
||||
let binarySha256 = null, binaryBytes = null, archiveSha256 = null, archiveMemberBytes = null;
|
||||
try {
|
||||
try { lstatSync(staged); throw new Error("staged dist/bin runnerd would override the admitted debug executable"); }
|
||||
catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
let directory = repositoryRoot;
|
||||
for (const part of selectedPath.split("/").slice(0, -1)) {
|
||||
directory = join(directory, part);
|
||||
if (!lstatSync(directory).isDirectory()) throw new Error("selected runnerd ancestor must be a real directory");
|
||||
}
|
||||
const binary = join(repositoryRoot, selectedPath), stat = regularFile(binary);
|
||||
if (process.platform !== "win32" && !(stat.mode & 0o111)) throw new Error("selected runnerd is not executable");
|
||||
binaryBytes = stat.size;
|
||||
binarySha256 = archiveDigest(binary);
|
||||
if (hosted) {
|
||||
if (environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== sourceSha || !/^[a-f0-9]{40}$/.test(sourceSha ?? ""))
|
||||
throw new Error("trusted hosted target SHA does not match admitted Git HEAD");
|
||||
if (![environment.GITHUB_RUN_ID, environment.GITHUB_RUN_ATTEMPT].every(value => typeof value === "string" && value === value.trim() && /^[1-9]\d*$/.test(value)))
|
||||
throw new Error("trusted hosted workflow run and attempt are unavailable");
|
||||
const hydration = inspectNativeCompletionBuildHydration(repositoryRoot);
|
||||
if (!hydration.verified) throw new Error(`hosted build archive is unverified: ${hydration.errors.join("; ")}`);
|
||||
archiveSha256 = hydration.archiveSha256;
|
||||
const archive = join(repositoryRoot, hydrationDirectory, archiveName);
|
||||
const tar = args => spawnSync("tar", args, { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024 });
|
||||
const names = tar(["-tzf", archive]);
|
||||
if (names.status !== 0) throw new Error("hosted build archive cannot be listed");
|
||||
const matching = names.stdout.split(/\r?\n/).filter(name => name && posix.normalize(name.replace(/^\/+/, "")) === selectedPath);
|
||||
if (matching.length !== 1 || matching[0] !== selectedPath)
|
||||
throw new Error("hosted archive must contain exactly one canonical selected runnerd member");
|
||||
const details = tar(["-tvzf", archive, selectedPath]);
|
||||
const rows = details.stdout.trim().split(/\r?\n/);
|
||||
if (details.status !== 0 || rows.length !== 1 || !rows[0].startsWith("-") || !rows[0].endsWith(` ${selectedPath}`))
|
||||
throw new Error("hosted selected runnerd member must be a regular file");
|
||||
// Keep daemon bytes out of JavaScript memory; the selected executable supplies the exact expected size.
|
||||
const temporary = mkdtempSync(join(tmpdir(), "native-completion-runnerd-member-")), member = join(temporary, "runnerd");
|
||||
let fd;
|
||||
try {
|
||||
fd = openSync(member, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW ?? 0), 0o600);
|
||||
const extracted = spawnSync("tar", ["-xOzf", archive, selectedPath], { timeout: 120_000, stdio: ["ignore", fd, "pipe"] });
|
||||
archiveMemberBytes = fstatSync(fd).size;
|
||||
closeSync(fd); fd = undefined;
|
||||
if (extracted.status !== 0 || archiveMemberBytes !== binaryBytes || archiveDigest(member) !== binarySha256)
|
||||
throw new Error("selected runnerd byte count or hash differs from the verified hosted archive member");
|
||||
} finally {
|
||||
if (fd !== undefined) closeSync(fd);
|
||||
rmSync(temporary, { recursive: true, force: true });
|
||||
}
|
||||
if (archiveDigest(archive) !== archiveSha256) throw new Error("hosted build archive changed during runnerd verification");
|
||||
}
|
||||
} catch (error) { errors.push(`runnerd provenance ${error.message}`); }
|
||||
return { schema: "paperclip.native-completion-runnerd-provenance.v1", mode: hosted ? "trusted_hosted_archive" : "fresh_local_build",
|
||||
passed: errors.length === 0, selectedPath, binarySha256, binaryBytes, archiveMemberBytes, sourceSha, sourceFingerprint, archiveSha256,
|
||||
workflowRunId: hosted ? environment.GITHUB_RUN_ID ?? null : null,
|
||||
workflowRunAttempt: hosted ? environment.GITHUB_RUN_ATTEMPT ?? null : null,
|
||||
artifactName: hosted ? `runner-e2e-build-${sourceSha}-${environment.GITHUB_RUN_ID}-${environment.GITHUB_RUN_ATTEMPT}` : null,
|
||||
errors };
|
||||
}
|
||||
|
||||
/** Preserve tracked-source cleanliness; allow only the verified workflow download. */
|
||||
export function inspectNativeCompletionSourceMetadata({ repositoryRoot, sourceFiles, baseSha, variant, shallowParentAnchors }) {
|
||||
const git = (args, raw = false) => spawnSync("git", ["--no-replace-objects", ...args], {
|
||||
cwd: repositoryRoot, encoding: raw ? undefined : "utf8", timeout: 30_000,
|
||||
});
|
||||
const head = git(["rev-parse", "--verify", "HEAD"]), sha = head.status === 0 ? head.stdout.trim() : null;
|
||||
const ancestor = git(["merge-base", "--is-ancestor", baseSha, "HEAD"]);
|
||||
const shallow = git(["rev-parse", "--is-shallow-repository"]), isShallow = shallow.status === 0 && shallow.stdout.trim() === "true";
|
||||
const errors = [];
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) errors.push("Git HEAD is unavailable or is not an immutable SHA1 commit");
|
||||
let layering = ancestor.status === 0, strategy = layering ? "full_ancestry" : null;
|
||||
const anchor = shallowParentAnchors?.[variant] ?? null;
|
||||
if (!layering && isShallow && anchor && sha) {
|
||||
const commit = git(["cat-file", "commit", "HEAD"], true);
|
||||
layering = commit.status === 0 && verifyNativeCompletionParentAnchor({ sha, rawCommit: commit.stdout, anchor });
|
||||
if (layering) strategy = "shallow_immediate_parent_anchor";
|
||||
}
|
||||
if (!layering) errors.push(isShallow
|
||||
? "shallow Git HEAD does not have the sole hash-verified immediate parent admitted for this source variant"
|
||||
: "full Git history does not prove the declared master base is an ancestor of HEAD");
|
||||
const tracked = git(["ls-files", "--error-unmatch", "--", ...sourceFiles]);
|
||||
const clean = git(["diff", "--quiet", "HEAD", "--"]);
|
||||
const status = git(["status", "--porcelain=v1", "-z", "--untracked-files=all"]);
|
||||
const entries = status.status === 0 ? status.stdout.split("\0").filter(Boolean) : [];
|
||||
const unexpected = entries.filter(entry => entry.slice(0, 3) !== "?? " || !hydrationPaths.includes(entry.slice(3)));
|
||||
const hydration = inspectNativeCompletionBuildHydration(repositoryRoot);
|
||||
if (tracked.status !== 0) errors.push("one or more admitted source paths are not tracked at HEAD");
|
||||
if (clean.status !== 0) errors.push("tracked source differs from HEAD");
|
||||
if (status.status !== 0) errors.push("Git worktree status is unavailable");
|
||||
if (unexpected.length) errors.push(`unexpected worktree entries: ${unexpected.join(", ")}`);
|
||||
errors.push(...hydration.errors);
|
||||
const immutable = tracked.status === 0 && clean.status === 0 && status.status === 0 && unexpected.length === 0 &&
|
||||
(!hydration.present || hydration.verified);
|
||||
const metadata = { schema: "paperclip.native-completion-source-metadata.v1",
|
||||
lineage: { strategy, shallow: isShallow, sha, baseSha, anchor: strategy === "shallow_immediate_parent_anchor" ? anchor : null },
|
||||
hydration: { present: hydration.present, verified: hydration.verified, archiveSha256: hydration.archiveSha256, checksumSha256: hydration.checksumSha256 },
|
||||
worktreeEntries: entries, errors };
|
||||
return { sha, layering, immutable, sourceMetadata: metadata, sourceMetadataErrors: errors,
|
||||
sourceMetadataFingerprint: sha256(JSON.stringify(metadata)) };
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { after, test } from "node:test";
|
||||
import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { inspectNativeCompletionSourceMetadata, inspectNativeCompletionBuildHydration, verifyNativeCompletionParentAnchor,
|
||||
inspectNativeCompletionRunnerd, NATIVE_COMPLETION_RUNNERD_PATH } from "./native-completion-git-source.mjs";
|
||||
|
||||
const roots = [];
|
||||
after(() => { for (const root of roots) rmSync(root, { recursive: true, force: true }); });
|
||||
const temporary = () => { const root = mkdtempSync(join(tmpdir(), "native-source-metadata-")); roots.push(root); return root; };
|
||||
const git = (root, args, input) => execFileSync("git", args, { cwd: root, input, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
const digest = bytes => createHash("sha256").update(bytes).digest("hex");
|
||||
const archiveName = "runner-e2e-build-bundle.tar.gz", checksumName = `${archiveName}.sha256`;
|
||||
function commit(root, tree, parents, message) {
|
||||
const raw = `tree ${tree}\n${parents.map(parent => `parent ${parent}\n`).join("")}author Fixture <fixture@example.invalid> 1 +0000\ncommitter Fixture <fixture@example.invalid> 1 +0000\n\n${message}\n`;
|
||||
return { raw, sha: git(root, ["hash-object", "-t", "commit", "-w", "--stdin"], raw) };
|
||||
}
|
||||
function fixture(shallow = false, parentCount = 1) {
|
||||
const source = temporary(); git(source, ["init", "--quiet"]);
|
||||
writeFileSync(join(source, "source.txt"), "Immutable admitted source\n");
|
||||
const blob = git(source, ["hash-object", "-w", "source.txt"]);
|
||||
const tree = git(source, ["mktree"], `100644 blob ${blob}\tsource.txt\n`);
|
||||
const base = commit(source, tree, [], "Admitted base"), anchor = commit(source, tree, [base.sha], "Admitted native parent");
|
||||
const other = commit(source, tree, [base.sha], "Different parent");
|
||||
const head = commit(source, tree, parentCount === 2 ? [anchor.sha, other.sha] : [anchor.sha], "Native-only repair");
|
||||
git(source, ["update-ref", "refs/heads/fixture", head.sha]); git(source, ["checkout", "--quiet", "--force", "fixture"]);
|
||||
let root = source;
|
||||
if (shallow) {
|
||||
root = join(temporary(), "checkout");
|
||||
git(source, ["clone", "--quiet", "--depth", "1", "--branch", "fixture", pathToFileURL(source).href, root]);
|
||||
}
|
||||
const input = { repositoryRoot: root, sourceFiles: ["source.txt"], baseSha: base.sha, variant: "candidate",
|
||||
shallowParentAnchors: { candidate: anchor.sha, historical: other.sha } };
|
||||
return { root, source, base, anchor, other, head, input };
|
||||
}
|
||||
function hydrate(root, bytes = Buffer.from("Exact downloaded build archive bytes")) {
|
||||
const directory = join(root, "runner-e2e-build"); mkdirSync(directory);
|
||||
writeFileSync(join(directory, archiveName), bytes);
|
||||
writeFileSync(join(directory, checksumName), `${digest(bytes)} ${archiveName}\n`);
|
||||
return directory;
|
||||
}
|
||||
test("native hosted-source metadata retains full ancestry and clean sources", () => {
|
||||
const f = fixture(), result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.layering, true); assert.equal(result.immutable, true); assert.deepEqual(result.sourceMetadataErrors, []);
|
||||
assert.equal(result.sourceMetadata.lineage.strategy, "full_ancestry");
|
||||
});
|
||||
test("native hosted-source metadata accepts an actual shallow clone with the exact raw parent anchor", () => {
|
||||
const f = fixture(true); hydrate(f.root);
|
||||
assert.equal(git(f.root, ["rev-parse", "--is-shallow-repository"]), "true");
|
||||
assert.throws(() => git(f.root, ["merge-base", "--is-ancestor", f.base.sha, "HEAD"]));
|
||||
const result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.sha, f.head.sha); assert.equal(result.layering, true); assert.equal(result.immutable, true);
|
||||
assert.deepEqual(result.sourceMetadataErrors, []);
|
||||
assert.equal(result.sourceMetadata.lineage.strategy, "shallow_immediate_parent_anchor");
|
||||
assert.equal(result.sourceMetadata.lineage.anchor, f.anchor.sha);
|
||||
assert.equal(result.sourceMetadata.hydration.verified, true);
|
||||
});
|
||||
test("native hosted-source metadata requires the variant-specific shallow anchor", () => {
|
||||
const f = fixture(true), result = inspectNativeCompletionSourceMetadata({ ...f.input, variant: "historical" });
|
||||
assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified immediate parent")));
|
||||
});
|
||||
test("native hosted-source metadata never substitutes an anchor for missing full-clone ancestry", () => {
|
||||
const f = fixture(), result = inspectNativeCompletionSourceMetadata({ ...f.input, baseSha: "f".repeat(40) });
|
||||
assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("full Git history")));
|
||||
});
|
||||
test("native hosted-source metadata rejects a shallow merge even with the admitted parent", () => {
|
||||
const f = fixture(true, 2), result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified")));
|
||||
});
|
||||
test("native hosted-source metadata cryptographically binds all raw HEAD bytes and the sole parent", () => {
|
||||
const f = fixture();
|
||||
assert.equal(verifyNativeCompletionParentAnchor({ sha: f.head.sha, rawCommit: f.head.raw, anchor: f.anchor.sha }), true);
|
||||
for (const input of [
|
||||
{ sha: "f".repeat(40), rawCommit: f.head.raw, anchor: f.anchor.sha },
|
||||
{ sha: f.head.sha, rawCommit: `${f.head.raw}altered`, anchor: f.anchor.sha },
|
||||
{ sha: f.head.sha, rawCommit: f.head.raw, anchor: f.other.sha },
|
||||
{ sha: f.base.sha, rawCommit: f.base.raw, anchor: f.anchor.sha },
|
||||
{ sha: null, rawCommit: f.head.raw, anchor: f.anchor.sha },
|
||||
]) assert.equal(verifyNativeCompletionParentAnchor(input), false);
|
||||
});
|
||||
test("native hosted-source metadata rejects dirty tracked and untracked admitted source", () => {
|
||||
const f = fixture(); hydrate(f.root); writeFileSync(join(f.root, "source.txt"), "Changed admitted source\n");
|
||||
let result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.includes("tracked source differs from HEAD"));
|
||||
git(f.root, ["checkout", "HEAD", "--", "source.txt"]); writeFileSync(join(f.root, "extra-source.mjs"), "Unexpected code");
|
||||
result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("extra-source.mjs")));
|
||||
assert.equal(inspectNativeCompletionSourceMetadata({ ...f.input, sourceFiles: ["extra-source.mjs"] }).immutable, false);
|
||||
});
|
||||
test("native hosted-source metadata verifies only the exact two downloaded regular files", () => {
|
||||
const f = fixture(), directory = hydrate(f.root), first = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(first.immutable, true); assert.equal(first.sourceMetadata.hydration.verified, true);
|
||||
assert.equal(inspectNativeCompletionSourceMetadata(f.input).sourceMetadataFingerprint, first.sourceMetadataFingerprint);
|
||||
writeFileSync(join(directory, archiveName), "Different verified archive");
|
||||
writeFileSync(join(directory, checksumName), `${digest("Different verified archive")} ${archiveName}\n`);
|
||||
const changed = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(changed.immutable, true); assert.notEqual(changed.sourceMetadataFingerprint, first.sourceMetadataFingerprint);
|
||||
});
|
||||
const negatives = [
|
||||
["missing checksum", (root, directory) => rmSync(join(directory, checksumName))],
|
||||
["checksum mismatch", (root, directory) => writeFileSync(join(directory, archiveName), "Unverified replacement")],
|
||||
["extra file", (root, directory) => writeFileSync(join(directory, "extra.mjs"), "Unexpected code")],
|
||||
["extra nested directory", (root, directory) => mkdirSync(join(directory, "nested"))],
|
||||
["malformed checksum", (root, directory) => writeFileSync(join(directory, checksumName), "malformed")],
|
||||
["wrong checksum filename", (root, directory) => writeFileSync(join(directory, checksumName), `${"a".repeat(64)} other.tar.gz\n`)],
|
||||
["multiple checksum records", (root, directory) => writeFileSync(join(directory, checksumName), readFileSync(join(directory, checksumName), "utf8").repeat(2))],
|
||||
["symlink archive", (root, directory) => { rmSync(join(directory, archiveName)); symlinkSync(join(root, "source.txt"), join(directory, archiveName)); }],
|
||||
["symlink checksum", (root, directory) => { rmSync(join(directory, checksumName)); symlinkSync(join(root, "source.txt"), join(directory, checksumName)); }],
|
||||
["symlink directory", (root, directory) => { rmSync(directory, { recursive: true }); symlinkSync(root, directory); }],
|
||||
];
|
||||
for (const [name, mutate] of negatives) test(`native hosted-source metadata rejects ${name} hydration`, () => {
|
||||
const f = fixture(), directory = hydrate(f.root); mutate(f.root, directory);
|
||||
const hydration = inspectNativeCompletionBuildHydration(f.root), result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(hydration.verified, false); assert.ok(hydration.errors.length > 0);
|
||||
assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("build hydration")));
|
||||
});
|
||||
test("native hosted-source metadata rejects unrelated dirty trees despite verified hydration", () => {
|
||||
const f = fixture(); hydrate(f.root); mkdirSync(join(f.root, "untrusted")); writeFileSync(join(f.root, "untrusted", "anything"), "extra");
|
||||
const result = inspectNativeCompletionSourceMetadata(f.input);
|
||||
assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("untrusted/anything")));
|
||||
});
|
||||
|
||||
function runnerdFixture() {
|
||||
const root = temporary(), binary = join(root, NATIVE_COMPLETION_RUNNERD_PATH);
|
||||
mkdirSync(join(binary, ".."), { recursive: true }); writeFileSync(binary, "Exact built runnerd fixture bytes"); chmodSync(binary, 0o755);
|
||||
const directory = join(root, "runner-e2e-build"); mkdirSync(directory);
|
||||
function pack(members = [NATIVE_COMPLETION_RUNNERD_PATH]) {
|
||||
execFileSync("tar", ["-czf", join(directory, archiveName), ...members], { cwd: root });
|
||||
writeFileSync(join(directory, checksumName), `${digest(readFileSync(join(directory, archiveName)))} ${archiveName}\n`);
|
||||
}
|
||||
pack();
|
||||
const input = { repositoryRoot: root, sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), environment: {
|
||||
GITHUB_ACTIONS: "true", PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "2" } };
|
||||
return { root, binary, directory, pack, input };
|
||||
}
|
||||
test("native hosted runnerd proof binds the actual selected debug executable to the trusted same-run archive", () => {
|
||||
const f = runnerdFixture(), proof = inspectNativeCompletionRunnerd(f.input);
|
||||
assert.equal(proof.passed, true); assert.equal(proof.mode, "trusted_hosted_archive");
|
||||
assert.equal(proof.selectedPath, NATIVE_COMPLETION_RUNNERD_PATH);
|
||||
assert.equal(proof.binarySha256, digest(readFileSync(f.binary)));
|
||||
assert.equal(proof.binaryBytes, readFileSync(f.binary).length); assert.equal(proof.archiveMemberBytes, proof.binaryBytes);
|
||||
assert.equal(proof.artifactName, `runner-e2e-build-${f.input.sourceSha}-12345-2`);
|
||||
assert.deepEqual(proof.errors, []);
|
||||
const local = inspectNativeCompletionRunnerd({ ...f.input, environment: {} });
|
||||
assert.equal(local.passed, true); assert.equal(local.mode, "fresh_local_build");
|
||||
assert.equal(local.archiveSha256, null); assert.equal(local.workflowRunId, null);
|
||||
assert.equal(local.binaryBytes, readFileSync(f.binary).length); assert.equal(local.archiveMemberBytes, null);
|
||||
});
|
||||
for (const [name, mutate] of [
|
||||
["wrong trusted source SHA", f => { f.input.environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA = "c".repeat(40); }],
|
||||
["missing workflow run", f => { delete f.input.environment.GITHUB_RUN_ID; }],
|
||||
["malformed workflow attempt", f => { f.input.environment.GITHUB_RUN_ATTEMPT = "../2"; }],
|
||||
["newline in workflow run", f => { f.input.environment.GITHUB_RUN_ID = "12345\n"; }],
|
||||
["unverified archive", f => { writeFileSync(join(f.directory, archiveName), "Replacement"); }],
|
||||
["different selected executable", f => { writeFileSync(f.binary, "Stale binary"); }],
|
||||
["missing selected executable", f => { rmSync(f.binary); }],
|
||||
["nonexecutable selected binary", f => { chmodSync(f.binary, 0o644); }],
|
||||
["same-size different binary hash", f => { writeFileSync(f.binary, Buffer.alloc(readFileSync(f.binary).length, 0x58)); }],
|
||||
["symlink selected executable", f => { const bytes = readFileSync(f.binary); rmSync(f.binary); writeFileSync(join(f.root, "other"), bytes); symlinkSync(join(f.root, "other"), f.binary); }],
|
||||
["symlink executable ancestor", f => { const bytes = readFileSync(f.binary), directory = join(f.binary, ".."); rmSync(directory, { recursive: true }); mkdirSync(join(f.root, "other")); writeFileSync(join(f.root, "other", "paperclip-runnerd"), bytes); chmodSync(join(f.root, "other", "paperclip-runnerd"), 0o755); symlinkSync(join(f.root, "other"), directory); }],
|
||||
["staged executable override", f => { const staged = join(f.root, "packages/paperclip-runner/dist/bin"); mkdirSync(staged, { recursive: true }); writeFileSync(join(staged, "paperclip-runnerd"), "Unexpected staged binary"); }],
|
||||
["duplicate canonical archive members", f => { f.pack([NATIVE_COMPLETION_RUNNERD_PATH, NATIVE_COMPLETION_RUNNERD_PATH]); }],
|
||||
["noncanonical archive member", f => { f.pack([`./${NATIVE_COMPLETION_RUNNERD_PATH}`]); }],
|
||||
["missing archive member", f => { writeFileSync(join(f.root, "other"), "Other"); f.pack(["other"]); }],
|
||||
["symlink archive member", f => { rmSync(f.binary); writeFileSync(join(f.root, "other"), "Other"); symlinkSync(join(f.root, "other"), f.binary); f.pack(); rmSync(f.binary); writeFileSync(f.binary, "Other"); chmodSync(f.binary, 0o755); }],
|
||||
]) test(`native hosted runnerd proof rejects ${name}`, () => {
|
||||
const f = runnerdFixture(); mutate(f);
|
||||
const proof = inspectNativeCompletionRunnerd(f.input);
|
||||
assert.equal(proof.passed, false); assert.ok(proof.errors.length > 0);
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { rehydrateRunnerdItemNotification } from "../../packages/paperclip-runner/src/live/runnerd-codex-transport.js";
|
||||
import { gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js";
|
||||
type Row = Record<string, unknown>;
|
||||
function sample(blocked = true, compatibility = false): NativeCompletionObservation {
|
||||
@@ -14,7 +16,7 @@ function sample(blocked = true, compatibility = false): NativeCompletionObservat
|
||||
runs: [{ id: "run", nativeIssueId: "issue", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", resultJson: { nativeResult: result } }],
|
||||
comments: [{ createdByRunId: "run", authorAgentId: "agent", body }],
|
||||
initial: { issueIds: [], agentIds: ["agent"] }, state: { issueIds: ["issue"], agentIds: ["agent"], documentCount: blocked ? 0 : 1, interactionCount: 0 }, workspaceChanged: false,
|
||||
events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool" } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result),
|
||||
events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool", name: tool } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result),
|
||||
compatibility ? event(3, "item.completed", { kind: "tool_result", item: { type: "tool_result", status: "completed", id: "tool" } }) : event(3, "tool.execution.completed", { name: tool, status: "completed", executionId: "tool" }),
|
||||
event(4, "item.completed", { kind: "agentMessage", channel: "final", item: { type: "agentMessage", phase: "final_answer", text: body, channel: "final" } }),
|
||||
event(5, "run.result.accepted", { result }, "control_plane"), event(6, "run.terminal", { runTerminalState: "succeeded", turnTerminalState: "completed" }, "control_plane")],
|
||||
@@ -23,6 +25,22 @@ function sample(blocked = true, compatibility = false): NativeCompletionObservat
|
||||
function payload(input: NativeCompletionObservation, index: number): Row { return ((input.events[index]!.payload as Row).prpEvent as Row).payload as Row; }
|
||||
describe("native completion independent oracle", () => {
|
||||
it.each([[true, false], [true, true], [false, false], [false, true]])("accepts disposition %s compatibility %s with final before late control-plane acceptance", (blocked, compatibility) => expect(gradeNativeCompletion(sample(blocked, compatibility)).passed).toBe(true));
|
||||
it.each(["paperclip_finish", "paperclip_block"])("carries normalized %s identity through rehydration into the exact-call oracle", name => {
|
||||
// The Rust normalization calibration asserts these exact fixture bytes.
|
||||
const fixture = JSON.parse(readFileSync(new URL("./fixtures/native-completion/terminal-tool-carrier.json", import.meta.url), "utf8")) as {
|
||||
cases: Array<{ name: string; normalizedPayload: Row }>;
|
||||
};
|
||||
const normalized = fixture.cases.find(value => value.name === name)!.normalizedPayload;
|
||||
const started = rehydrateRunnerdItemNotification(normalized, "thread", "turn");
|
||||
expect(started.item).toMatchObject({ id: "terminal-call", type: "tool_call", name });
|
||||
const completed = rehydrateRunnerdItemNotification({ provider: "codex", itemId: "terminal-call", kind: "tool_result", status: "completed", channel: "detail", text: null }, "thread", "turn");
|
||||
const value = sample(name === "paperclip_block", true);
|
||||
payload(value, 0).item = started.item;
|
||||
payload(value, 2).item = completed.item;
|
||||
expect(gradeNativeCompletion(value).passed).toBe(true);
|
||||
(payload(value, 2).item as Row).id = "different-call";
|
||||
expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false);
|
||||
});
|
||||
it("accepts authoritative acceptance before the provider final", () => {
|
||||
const value = sample();
|
||||
const events = [...value.events];
|
||||
@@ -40,6 +58,13 @@ describe("native completion independent oracle", () => {
|
||||
else (payload(value, 2).item as Row).id = "unmatched";
|
||||
expect(gradeNativeCompletion(value).passed).toBe(false);
|
||||
});
|
||||
it.each(["missing-name", "unrelated-named-tool", "contradictory-result-name"])("rejects compatibility finishing identity %s despite a matching ID", name => {
|
||||
const value = sample(true, true);
|
||||
if (name === "missing-name") delete (payload(value, 0).item as Row).name;
|
||||
if (name === "unrelated-named-tool") (payload(value, 0).item as Row).name = "write_document";
|
||||
if (name === "contradictory-result-name") (payload(value, 2).item as Row).name = "write_document";
|
||||
expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false);
|
||||
});
|
||||
it.each(["extra-run", "retry", "continuation", "wrong-account", "wrong-agent", "wrong-disposition", "punctuated-action", "wrong-scope", "missing-final", "summary-fallback", "pre-tool-final", "post-admission-call", "missing-acceptance", "runner-acceptance", "failed-terminal", "event-gap", "binding-mismatch", "extra-task", "extra-agent", "extra-document", "interaction", "changed-workspace", "process-call", "marker-only", "contradiction", "wrong-reply-run"])("rejects %s", name => {
|
||||
const value = sample(); const run = value.runs[0]!;
|
||||
if (name === "extra-run") value.runs = [run, { ...run, id: "extra" }];
|
||||
|
||||
@@ -83,6 +83,8 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) {
|
||||
|| event.eventType === "item.completed" && start.eventType === "item.started"
|
||||
&& (started.kind === "tool_call" || startedItem.type === "tool_call")
|
||||
&& typeof item.id === "string" && item.id.length > 0 && item.id === startedItem.id
|
||||
&& terminalName(startedItem.name, expectedTool)
|
||||
&& (item.name === undefined || item.name === null || terminalName(item.name, expectedTool))
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -128,6 +130,6 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) {
|
||||
check("no-deployment-or-file-work", !input.workspaceChanged && !forbidden,
|
||||
"The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed.");
|
||||
}
|
||||
return { schema: "paperclip.native-completion-observation.v1", passed: checks.every(value => value.passed), checks,
|
||||
return { schema: "paperclip.native-completion-observation.v2", passed: checks.every(value => value.passed), checks,
|
||||
limitations: ["Exact provider feedback identity/consumption is not measured by the public sequence."] };
|
||||
}
|
||||
@@ -7,6 +7,10 @@ export const NATIVE_COMPLETION_SOURCE_CONTRACT = {
|
||||
"schema": "paperclip.native-completion-source-contract.v1",
|
||||
"baseSha": "59c07ede72dc08b8aba149a01cc11e0b7a204621",
|
||||
"archiveSha": "9138f570c341c251a5727c32d6615ce238bc8e03",
|
||||
"shallowParentAnchors": {
|
||||
"candidate": "e18c2cf9e96a4d31acb6d03ce918dfe223107d3f",
|
||||
"historical": "459455acb11a012a97ad1b4afb77a1dc024a88bc"
|
||||
},
|
||||
"variants": {
|
||||
"candidate": {
|
||||
"packages/paperclip-runner/src/contracts/completion-result.ts": "bd4bb79d2be5c4ee3765df67dfb4da98ef95bc532cac872be40cb41a6bff6cda",
|
||||
@@ -61,6 +65,9 @@ export const NATIVE_COMPLETION_FIXTURE_FILES = [
|
||||
"tests/runner-e2e/native-completion-admission.test.ts",
|
||||
"tests/runner-e2e/native-completion-checks.mjs",
|
||||
"tests/runner-e2e/native-completion-checks.test.mjs",
|
||||
"tests/runner-e2e/native-completion-git-source.mjs",
|
||||
"tests/runner-e2e/native-completion-git-source.test.mjs",
|
||||
"tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json",
|
||||
"tests/runner-e2e/native-completion-source-contract.mjs",
|
||||
"tests/runner-e2e/native-completion-source-contract.test.mjs",
|
||||
"tests/runner-e2e/automatic-retry.ts",
|
||||
@@ -108,6 +115,9 @@ export const NATIVE_COMPLETION_SOURCE_FILES = [...new Set([
|
||||
"packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts",
|
||||
"packages/adapters/codex-local/src/index.ts", "packages/adapters/claude-local/src/index.ts",
|
||||
"packages/paperclip-runner/package.json", "tsconfig.base.json",
|
||||
"packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs",
|
||||
"packages/paperclip-runner/runner/Cargo.toml", "packages/paperclip-runner/runner/Cargo.lock",
|
||||
"packages/paperclip-runner/runner/crates/runner-core/Cargo.toml",
|
||||
])].sort();
|
||||
|
||||
export const nativeSourceSha256 = bytes => createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -50,6 +50,15 @@ test("native source contract rejects an ambiguous identical variant map", () =>
|
||||
const f = fixture(); f.contract.variants.historical = f.contract.variants.candidate;
|
||||
assert.equal(nativeCompletionSourceFingerprint(f.read, f.contract).variant, null);
|
||||
});
|
||||
test("native source contract binds common Rust carrier source and the shared behavioral projection fixture", () => {
|
||||
const f = fixture(), before = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
f.bytes.set("packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs", Buffer.from("Changed carrier"));
|
||||
const carrier = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.equal(carrier.variant, "candidate"); assert.notEqual(carrier.fingerprint, before.fingerprint);
|
||||
f.bytes.set("tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json", Buffer.from("Changed projection"));
|
||||
const projection = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.notEqual(projection.fingerprint, carrier.fingerprint); assert.notEqual(projection.fixtureFingerprint, carrier.fixtureFingerprint);
|
||||
});
|
||||
test("native source contract binds exactly five production and six variant assertion files", () => {
|
||||
const files = Object.keys(original.variants.candidate);
|
||||
assert.equal(files.length, 11); assert.equal(files.filter(file => file.endsWith(".test.ts")).length, 6);
|
||||
@@ -57,5 +66,9 @@ test("native source contract binds exactly five production and six variant asser
|
||||
assert.ok(files.every(file => original.variants.candidate[file] !== original.variants.historical[file]));
|
||||
assert.equal(original.baseSha, "59c07ede72dc08b8aba149a01cc11e0b7a204621");
|
||||
assert.equal(original.archiveSha, "9138f570c341c251a5727c32d6615ce238bc8e03");
|
||||
assert.deepEqual(original.shallowParentAnchors, {
|
||||
candidate: "e18c2cf9e96a4d31acb6d03ce918dfe223107d3f",
|
||||
historical: "459455acb11a012a97ad1b4afb77a1dc024a88bc",
|
||||
});
|
||||
assert.ok(!NATIVE_COMPLETION_SOURCE_FILES.some(file => file.includes("stock-harness") || file.endsWith("issue-documents.md")));
|
||||
});
|
||||
Reference in new issue
Block a user