diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 29a0316692f..0bdf4a0e881 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -472,4 +472,12 @@ The receipt reader fails closed when the production native receipt projection is missing, partial, duplicated or outside its bounds (including 256 KiB for the whole native rawOutput, which may repeat text). This oracle depends on the new production semantic-receipt contract and does not regrade earlier failed runs. +The bridge call hash identifies the invocation and its native receipt. It is not +inferred from `run.result.proposed.itemId`, which can identify the run instead. +Exactly one proposed body must hash to the receipt input and equal exactly one +accepted body; identical duplicate proposals or receipts still fail. The shell +result is one separate complete read lifecycle: pending and any progress name the +started shell, and its successful terminal also names the original command. A +completed-only read or a second shell read cannot inherit that exemption. + Denial case version 5 and denial settlement schema v3 are unchanged. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 47ea91a3964..dcd022aeb06 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1520,4 +1520,12 @@ The receipt reader fails closed when the production native receipt projection is missing, partial, duplicated or outside its bounds (including 256 KiB for the whole native rawOutput, which may repeat text). This oracle depends on the new production semantic-receipt contract and does not regrade earlier failed runs. +The bridge call hash identifies the invocation and its native receipt. It is not +inferred from `run.result.proposed.itemId`, which can identify the run instead. +Exactly one proposed body must hash to the receipt input and equal exactly one +accepted body; identical duplicate proposals or receipts still fail. The shell +result is one separate complete read lifecycle: pending and any progress name the +started shell, and its successful terminal also names the original command. A +completed-only read or a second shell read cannot inherit that exemption. + Denial case version 5 and denial settlement schema v3 are unchanged. diff --git a/tests/runner-e2e/copilot-semantic-evidence.test.ts b/tests/runner-e2e/copilot-semantic-evidence.test.ts index e142916daf2..165c6b2f20e 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.test.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.test.ts @@ -16,7 +16,7 @@ function notice(seq: number, toolCallId: string, status: string, fields: any = { return row(seq, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "copilot_tool_evidence_v1", scope: "turn", provenance: { method: "session/update", eventType: "tool", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "tool", toolCallId, status, ...fields }).map(([name, value]) => ({ name, value: String(value) })) }); } function fixture() { - const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "3" }), + const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "item-run" }), row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v1", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }), notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }), row(6, "turn.completed", {}), row(7, "run.result.accepted", { result: structuredClone(result) }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1 })]; @@ -51,7 +51,7 @@ describe("Copilot semantic completion public-event oracle", () => { }); it.each(["failed", "cancelled", "interrupted"])("rejects %s terminal", status => { const f = fixture(); f.rows[5]!.eventType = `turn.${status}`; frame(f.rows[5]!).eventType = `turn.${status}`; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); it("rejects changed accepted body, forged call ID, wrong summary and rejected result", () => { - for (const mutate of [(f: ReturnType) => { frame(f.rows[6]!).payload.result.completionClaim.remainingWork.push("unfinished"); }, (f: ReturnType) => { frame(f.rows[2]!).itemId = "4"; }, (f: ReturnType) => { f.expected.summary = "other"; }, (f: ReturnType) => { f.rows.push(row(8, "run.result.rejected", {})); }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } + for (const mutate of [(f: ReturnType) => { frame(f.rows[6]!).payload.result.completionClaim.remainingWork.push("unfinished"); }, (f: ReturnType) => { setField(f.rows[3], "callIdentitySha256", hash("foreign-call")); }, (f: ReturnType) => { f.expected.summary = "other"; }, (f: ReturnType) => { f.rows.push(row(8, "run.result.rejected", {})); }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } }); it("rejects duplicate/unknown/oversized receipt details and partial native fields", () => { for (const mutate of [(r: any) => frame(r).payload.details.push({ name: "secret", value: "untrusted" }), (r: any) => frame(r).payload.details.push(frame(r).payload.details[0]), (r: any) => setField(r, "operationId", "x".repeat(300))]) { const f = fixture(); mutate(f.rows[3]); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } @@ -60,12 +60,61 @@ describe("Copilot semantic completion public-event oracle", () => { it("rejects mutation metadata, extra lifecycle, reordered receipt and source sequence drift", () => { for (const mutate of [(f: ReturnType) => frame(f.rows[1]!).payload.details.push({ name: "operation", value: "edit" }), (f: ReturnType) => f.rows.push(notice(8, "finish-native", "pending")), (f: ReturnType) => { f.rows[3]!.seq = 8; }, (f: ReturnType) => { frame(f.rows[3]!).sourceSeq = 20; f.rows[3]!.sourceSeq = 20; }, (f: ReturnType) => { frame(f.rows[1]!).sourceSeq = 20; f.rows[1]!.sourceSeq = 20; }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } }); - it("allows only exact command and correlated finish; unknown reads/edits/delegation remain failures", () => { - const f = fixture(), proof = readCopilotSemanticCompletion(f.rows, f.expected), notices = readCopilotToolEvidence(f.rows, "run"); - expect(onlyCopilotAttachedOperations(notices, f.expected.command, proof)).toBe(true); - for (const operation of [undefined, "read", "edit", "execute"] as const) expect(onlyCopilotAttachedOperations([...notices, { ...f.expected.command, toolCallId: "extra", operation }], f.expected.command, proof)).toBe(false); - expect(onlyCopilotAttachedOperations(notices, f.expected.command, { ...proof, nativeToolCallId: "command" })).toBe(false); - expect(onlyCopilotAttachedOperations([{ ...notices[1]!, sessionId: "foreign" }], f.expected.command, proof)).toBe(false); + it("does not treat canonical run-level itemId as semantic call identity", () => { + for (const itemId of ["item-run", "3", "unrelated-item", null]) { + const f = fixture(); frame(f.rows[2]!).itemId = itemId; + expect(readCopilotSemanticCompletion(f.rows, f.expected).callIdentitySha256).toBe(receipt.callIdentitySha256); + } + }); + it("rejects a different finish input even when proposal and acceptance agree", () => { + const f = fixture(); frame(f.rows[2]!).payload.completionClaim.criteria[0].evidenceRefs.push("different"); + frame(f.rows[6]!).payload.result = structuredClone(frame(f.rows[2]!).payload); + expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); + }); + it("accepts the retained shell-read ordering: pending has shellId only, completion gains command identity after finish", () => { + const f = attachedFixture(); + expect(f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")?.commandToolCallId).toBeUndefined(); + expect(onlyCopilotAttachedOperations(f.notices, f.command, f.proof)).toBe(true); + }); + it("accepts bounded in-progress shell updates without inventing terminal command identity", () => { + const f = attachedFixture(), pending = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")!; + f.notices.push({ ...pending, status: "in_progress", seq: 53 }); + expect(onlyCopilotAttachedOperations(f.notices, f.command, f.proof)).toBe(true); + }); + it("rejects shell-read evidence from a different durable stream despite matching native identity", () => { + const f = attachedFixture(), r = f.rows.find(r => r.seq === 52)!; + frame(r).sourceInstanceId = "foreign"; r.sourceInstanceId = "foreign"; + expect(() => readCopilotSemanticCompletion(f.rows, { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: f.command, summary: "EXACT-MARKER" })).toThrow(); + }); + it.each(["missing-pending", "missing-terminal", "duplicate-pending", "duplicate-terminal", "extra-completed-only", "extra-complete-read", "wrong-shell", "wrong-command", "failed", "nonzero", "reordered", "late", "wrong-operation", "mutation", "read-path", "foreign-session", "foreign-turn", "foreign-run", "permission", "semantic-fields"])("rejects invalid shell-read lifecycle: %s", failure => { + const f = attachedFixture(), start = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")!, end = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "completed")!; + if (failure === "missing-pending") f.notices.splice(f.notices.indexOf(start), 1); + if (failure === "missing-terminal") f.notices.splice(f.notices.indexOf(end), 1); + if (failure === "duplicate-pending") f.notices.push({ ...start, seq: 53 }); + if (failure === "duplicate-terminal") f.notices.push({ ...end, seq: 56 }); + if (failure === "extra-completed-only") f.notices.push({ ...end, toolCallId: "extra", seq: 57 }); + if (failure === "extra-complete-read") f.notices.push({ ...start, toolCallId: "extra", seq: 56 }, { ...end, toolCallId: "extra", seq: 57 }); + if (failure === "wrong-shell") start.shellId = "foreign"; + if (failure === "wrong-command") end.commandToolCallId = "foreign"; + if (failure === "failed") end.status = "failed"; + if (failure === "nonzero") end.exitCode = 1; + if (failure === "reordered") start.seq = 14; + if (failure === "late") end.seq = 61; + if (failure === "wrong-operation") start.operation = "execute"; + if (failure === "mutation") end.target = "file.txt"; + if (failure === "read-path") start.readTargetSha256 = `sha256:${"a".repeat(64)}`; + if (failure === "foreign-session") start.sessionId = "foreign"; + if (failure === "foreign-turn") start.turnId = "foreign"; + if (failure === "foreign-run") start.runId = "foreign"; + if (failure === "permission") start.stage = "permission_requested"; + if (failure === "semantic-fields") end.semanticCallIdentitySha256 = receipt.callIdentitySha256; + expect(onlyCopilotAttachedOperations(f.notices, f.command, f.proof)).toBe(false); + }); + it("rejects duplicate shell origins and arbitrary extra operations", () => { + const f = attachedFixture(); + const started = f.notices.find(n => n.shellState === "started")!; + expect(onlyCopilotAttachedOperations([...f.notices, { ...started, seq: 16 }], f.command, f.proof)).toBe(false); + for (const operation of [undefined, "read", "edit", "execute"] as const) expect(onlyCopilotAttachedOperations([...f.notices, { ...f.command, toolCallId: "extra", operation, seq: 17 }], f.command, f.proof)).toBe(false); }); it("keeps actual flow correlation before no-extra-operation gate and keeps visible marker independent", async () => { const source = await readFile(new URL("./copilot-protection-flow.ts", import.meta.url), "utf8"); @@ -75,3 +124,18 @@ describe("Copilot semantic completion public-event oracle", () => { expect(source).not.toContain('if (remote) check("no-extra-native-operation"'); }); }); + +// Mirrors retained native order: command pending→shell started; finish; then +// read_bash pending(shellId only)→completed(commandToolCallId), before turn end. +function attachedFixture() { + const f = fixture(); frame(f.rows[2]!).itemId = "3"; + for (const r of f.rows) { r.seq *= 10; if (frame(r).sourceKind === "runner") { r.sourceSeq *= 10; frame(r).sourceSeq *= 10; } } + const fields = { operation: "execute", commandSha256: `sha256:${"a".repeat(64)}`, mode: "async", detach: false }; + f.rows[0] = notice(10, "command", "pending", fields); + f.rows.push(notice(15, "command", "completed", { ...fields, shellId: "0", commandToolCallId: "command", shellState: "started" }), + notice(52, "shell-read", "pending", { operation: "read", shellId: "0" }), + notice(55, "shell-read", "completed", { operation: "read", shellId: "0", commandToolCallId: "command", shellState: "completed", exitCode: 0 })); + const notices = readCopilotToolEvidence(f.rows, "run"), command = notices.find(n => n.seq === 10)!; + const proof = readCopilotSemanticCompletion(f.rows, { ...f.expected, command }); + return { rows: f.rows, notices, command, proof }; +} diff --git a/tests/runner-e2e/copilot-semantic-evidence.ts b/tests/runner-e2e/copilot-semantic-evidence.ts index 1f0a7935d7a..6f1439dfd41 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.ts @@ -37,6 +37,13 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected return e; } frame(commandRows[0]!, "runner"); + // Every operation admitted later must belong to this durable stream, including + // shell reads whose command identity is only known on their terminal notice. + for (const n of notices) { + const rs = all.filter(r => r.seq === n.seq); + if (rs.length !== 1) fail("duplicate native row"); + frame(rs[0]!, "runner"); + } if (expected.command.runId !== expected.runId || expected.command.turnId !== expected.turnId || expected.command.sessionId !== expected.nativeSessionId) fail("foreign command"); const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1"); // This authored case asks for one finish, not arbitrary semantic actions. @@ -72,7 +79,11 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected || all.some(r => r.eventType === "run.result.rejected")) fail("missing, rejected or ambiguous completion"); const proposed = frame(proposedRows[0]!, "runner"), accepted = frame(acceptedRows[0]!, "control_plane"), terminal = frame(terminals[0]!, "runner"); const result = rec(proposed.payload), acceptedResult = rec(rec(accepted.payload).result); - if (!id(proposed.itemId) || hash(proposed.itemId) !== fields.callIdentitySha256 || digest(result) !== fields.inputSha256 + // Native correlation uses the invocation-captured callback call hash. The + // canonical proposal itemId may instead identify the run: never interpret it + // as a bridge call ID. Unique same-turn result bodies supply the independent + // acceptance proof, joined to that exact invocation's input digest. + if (digest(result) !== fields.inputSha256 || result.schema !== "paperclip.run_result.v1" || result.reportedWorkDisposition !== "done" || result.summary !== expected.summary || canonical(result) !== canonical(acceptedResult)) fail("canonical accepted result does not match exact finish"); const pendingFrame = frame(all.find(r => r.seq === pending[0]!.seq)!, "runner"); @@ -89,8 +100,37 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected /** Called only after the completion proof and bootstrap-read oracle succeed. */ export function onlyCopilotAttachedOperations(notices: readonly CopilotToolNotice[], command: CopilotToolNotice, proof: CopilotSemanticCompletionProof): boolean { - return proof.runId === command.runId && proof.turnId === command.turnId && proof.nativeSessionId === command.sessionId - && proof.nativeToolCallId !== command.toolCallId && notices.every(n => - n.runId === command.runId && n.sessionId === command.sessionId && n.turnId === command.turnId - && (n.toolCallId === command.toolCallId || n.commandToolCallId === command.toolCallId || n.toolCallId === proof.nativeToolCallId)); + if (proof.runId !== command.runId || proof.turnId !== command.turnId || proof.nativeSessionId !== command.sessionId + || proof.nativeToolCallId === command.toolCallId || notices.length > 2048 + || notices.some(n => n.runId !== command.runId || n.sessionId !== command.sessionId || n.turnId !== command.turnId + || !Number.isSafeInteger(n.seq) || n.seq < 1 || n.seq >= proof.turnCompletedSeq) + || new Set(notices.map(n => n.seq)).size !== notices.length) return false; + const groups = new Map(); + for (const n of notices) { const group = groups.get(n.toolCallId) ?? []; group.push(n); groups.set(n.toolCallId, group); } + // Exactly one command, one shell-result read, and the already-proven finish. + // A completed-only extra read cannot borrow the original command's identity. + if (groups.size !== 3 || !groups.has(proof.nativeToolCallId)) return false; + const complete = (group: CopilotToolNotice[]): boolean => { + group.sort((a, b) => a.seq - b.seq); + return group.length >= 2 && group[0]!.status === "pending" && group.at(-1)!.status === "completed" + && group.every((n, i) => n.stage === "tool" && (i === 0 ? n.status === "pending" + : i === group.length - 1 ? n.status === "completed" : n.status === "in_progress")); + }; + const semanticFree = (n: CopilotToolNotice) => n.semanticOperationId === undefined && n.semanticCallIdentitySha256 === undefined + && n.semanticInputSha256 === undefined && n.semanticResultSha256 === undefined && n.semanticOutcome === undefined; + const commandGroup = groups.get(command.toolCallId) ?? []; + if (!complete(commandGroup) || commandGroup[0]!.seq !== command.seq) return false; + const started = commandGroup.at(-1)!; + if (!id(started.shellId) || started.shellState !== "started" || started.commandToolCallId !== command.toolCallId + || commandGroup.some(n => n.operation !== "execute" || n.commandSha256 !== command.commandSha256 || n.mode !== "async" || n.detach !== false + || n.target !== undefined || n.readTargetSha256 !== undefined || n.exitCode !== undefined || !semanticFree(n) + || (n !== started && (n.shellId !== undefined || n.shellState !== undefined || n.commandToolCallId !== undefined)))) return false; + const shellGroup = [...groups].find(([key]) => key !== command.toolCallId && key !== proof.nativeToolCallId)![1]; + if (!complete(shellGroup)) return false; + const terminal = shellGroup.at(-1)!; + return shellGroup[0]!.seq > started.seq && terminal.seq < proof.turnCompletedSeq + && terminal.commandToolCallId === command.toolCallId && terminal.shellState === "completed" && terminal.exitCode === 0 + && shellGroup.every(n => n.operation === "read" && n.shellId === started.shellId && n.target === undefined && n.readTargetSha256 === undefined + && n.commandSha256 === undefined && n.mode === undefined && n.detach === undefined && semanticFree(n) + && (n === terminal || (n.commandToolCallId === undefined && n.shellState === undefined && n.exitCode === undefined))); }