diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 76dae43ee9e..2e594376f39 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -442,7 +442,7 @@ before Stop does not exercise this pending-callback boundary. This new suite has pure calibration and wiring tests, not a paid qualification result. Provider process death is not simulated by substituting the chat runner-worker crash hook. -### Copilot attached semantic completion evidence (suite 7) +### Copilot attached semantic completion evidence (suite 8) Attached settlement now requires one native `paperclip_finish` lifecycle joined to an invocation-captured, bounded Paperclip bridge receipt by exact call, input @@ -460,8 +460,21 @@ whole native rawOutput, which may repeat text). This oracle depends on the new production semantic-receipt contract and does not regrade earlier failed runs. The bridge call hash identifies the invocation and its native receipt. It is not inferred from `run.result.proposed.itemId`, which can identify the run instead. -Exactly one proposed body must hash to the receipt input and equal exactly one -accepted body; identical duplicate proposals or receipts still fail. The shell +The v2 receipt keeps the raw invocation input digest separate from the normalized +input digest captured by that same authenticated invocation after production +validation. Raw arguments may omit schema, artifacts and attention requests; +the production validator supplies those defaults. The oracle never reconstructs +or normalizes a body to make it match. The raw call/input/result hashes still join +the native lifecycle; its normalized input hash must also match the authority. +Exactly one proposed body must hash to that normalized digest and equal exactly +one accepted body; identical duplicate proposals or receipts still fail. + +Suite 8 writes `paperclip.e2e.copilot-semantic-completion.v2` and requires +`paperclip.semantic_tool_receipt.v2` under `paperclip_semantic_tool_receipt_v2`. +The authority has exactly eight bounded details. An explicit null normalized +digest is diagnostic only and cannot qualify a successful finish. Legacy v1 +receipts do not qualify fresh runs. This fixture depends on the corresponding +production receipt and durable-redaction fixes; it does not supply them. The shell result is one separate complete read lifecycle: pending and any progress name the started shell, and its successful terminal also names the original command. A completed-only read or a second shell read cannot inherit that exemption. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 489f11c7a2f..39c888e3200 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1473,7 +1473,7 @@ Provider death during a pending callback remains a separate uncovered case: the existing chat worker-crash hook targets a runner worker and cannot establish safe ownership of the native provider process in both environments. -### Copilot attached semantic completion evidence (suite 7) +### Copilot attached semantic completion evidence (suite 8) Attached settlement now requires one native `paperclip_finish` lifecycle joined to an invocation-captured, bounded Paperclip bridge receipt by exact call, input @@ -1491,8 +1491,21 @@ whole native rawOutput, which may repeat text). This oracle depends on the new production semantic-receipt contract and does not regrade earlier failed runs. The bridge call hash identifies the invocation and its native receipt. It is not inferred from `run.result.proposed.itemId`, which can identify the run instead. -Exactly one proposed body must hash to the receipt input and equal exactly one -accepted body; identical duplicate proposals or receipts still fail. The shell +The v2 receipt keeps the raw invocation input digest separate from the normalized +input digest captured by that same authenticated invocation after production +validation. Raw arguments may omit schema, artifacts and attention requests; +the production validator supplies those defaults. The oracle never reconstructs +or normalizes a body to make it match. The raw call/input/result hashes still join +the native lifecycle; its normalized input hash must also match the authority. +Exactly one proposed body must hash to that normalized digest and equal exactly +one accepted body; identical duplicate proposals or receipts still fail. + +Suite 8 writes `paperclip.e2e.copilot-semantic-completion.v2` and requires +`paperclip.semantic_tool_receipt.v2` under `paperclip_semantic_tool_receipt_v2`. +The authority has exactly eight bounded details. An explicit null normalized +digest is diagnostic only and cannot qualify a successful finish. Legacy v1 +receipts do not qualify fresh runs. This fixture depends on the corresponding +production receipt and durable-redaction fixes; it does not supply them. The shell result is one separate complete read lifecycle: pending and any progress name the started shell, and its successful terminal also names the original command. A completed-only read or a second shell read cannot inherit that exemption. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 2df004e0fd4..78b168f52ec 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1059,7 +1059,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 8, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v2", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-evidence.ts b/tests/runner-e2e/copilot-evidence.ts index d8272d96242..023e9b3292a 100644 --- a/tests/runner-e2e/copilot-evidence.ts +++ b/tests/runner-e2e/copilot-evidence.ts @@ -2,7 +2,7 @@ export interface CopilotToolNotice { runId: string; sessionId: string; turnId: string; toolCallId: string; observedAtMs: number; seq: number; - semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticResultSha256?: string; semanticOutcome?: "returned" | "error"; + semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticNormalizedInputSha256?: string | null; semanticResultSha256?: string; semanticOutcome?: "returned" | "error"; stage: "tool" | "permission_requested" | "permission_delivered"; status?: "pending" | "in_progress" | "completed" | "failed"; operation?: "edit" | "execute" | "read"; target?: string; requestId?: string; @@ -13,7 +13,7 @@ export interface CopilotToolNotice { const rec = (v: unknown): Record => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]"); const enums = { stage: ["tool", "permission_requested", "permission_delivered"], status: ["pending", "in_progress", "completed", "failed"], operation: ["edit", "execute", "read"], outcome: ["allow_once", "allow_always", "reject_once", "cancel"], mode: ["sync", "async"], shellState: ["started", "completed"] }; -const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]); +const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticNormalizedInputSha256", "semanticResultSha256", "semanticOutcome"]); export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: string): CopilotToolNotice[] { const result: CopilotToolNotice[] = []; for (const row of rows) { @@ -43,12 +43,14 @@ export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: if (fields.commandSha256 !== undefined) { if (!/^sha256:[a-f0-9]{64}$/u.test(fields.commandSha256)) throw new Error("Invalid command digest"); notice.commandSha256 = fields.commandSha256; } if (fields.shellId !== undefined) { if (!/^[A-Za-z0-9_.-]{1,80}$/u.test(fields.shellId)) throw new Error("Invalid shell identity"); notice.shellId = fields.shellId; } if (fields.exitCode !== undefined) { if (!/^-?\d{1,10}$/u.test(fields.exitCode) || !Number.isSafeInteger(Number(fields.exitCode))) throw new Error("Invalid exit code"); notice.exitCode = Number(fields.exitCode); } - const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]; + const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticNormalizedInputSha256", "semanticResultSha256", "semanticOutcome"]; if (semanticKeys.some(k => fields[k] !== undefined)) { if (!semanticKeys.every(k => fields[k] !== undefined) || fields.stage !== "tool" || !["completed", "failed"].includes(fields.status!) || !/^[A-Za-z0-9_.:-]{1,256}$/.test(fields.semanticOperationId!) || !["returned", "error"].includes(fields.semanticOutcome!) || ["semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256"].some(k => !/^[a-f0-9]{64}$/.test(fields[k]!))) throw new Error("Invalid semantic receipt fields"); + if (fields.semanticNormalizedInputSha256 !== "null" && !/^[a-f0-9]{64}$/.test(fields.semanticNormalizedInputSha256!)) throw new Error("Invalid normalized semantic digest"); for (const key of semanticKeys) notice[key] = fields[key]; + notice.semanticNormalizedInputSha256 = fields.semanticNormalizedInputSha256 === "null" ? null : fields.semanticNormalizedInputSha256; } result.push(notice as unknown as CopilotToolNotice); } diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 8c286a5917e..1db9b4c6250 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 7, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 8, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); diff --git a/tests/runner-e2e/copilot-semantic-evidence.test.ts b/tests/runner-e2e/copilot-semantic-evidence.test.ts index 165c6b2f20e..8f93315a371 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.test.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.test.ts @@ -3,11 +3,15 @@ import { readFile } from "node:fs/promises"; import { describe, expect, it } from "vitest"; import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; import { readCopilotToolEvidence } from "./copilot-evidence.js"; +import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js"; import { runnerSuites, suiteDefinitionHash } from "./catalog.js"; const hash = (v: string) => createHash("sha256").update(v).digest("hex"); const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v); -const result = { schema: "paperclip.run_result.v1", summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], artifacts: [], verification: [], attentionRequests: [] }; -const receipt = { schema: "paperclip.semantic_tool_receipt.v1", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" }; +const rawInput = { summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], verification: [] }; +const validation = validatePrpStructuredRunResult(rawInput); +if (!validation.ok) throw new Error("Semantic fixture input must pass the production validator"); +const result = validation.result; +const receipt = { schema: "paperclip.semantic_tool_receipt.v2", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(rawInput)), normalizedInputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" }; function row(seq: number, eventType: string, payload: any, overrides: any = {}) { const e = { schema: "paperclip.prp.event.v1", eventType, runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "source", sourceKind: "runner", sourceSeq: seq, emittedAt: "2026-09-30T12:00:00Z", payload, ...overrides }; return { companyId: "company", runId: "run", seq, eventType, sourceInstanceId: e.sourceInstanceId, sourceSeq: e.sourceSeq, payload: { prpEvent: e } }; @@ -17,8 +21,8 @@ function notice(seq: number, toolCallId: string, status: string, fields: any = { } function fixture() { const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "item-run" }), - row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v1", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }), - notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }), + row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v2", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }), + notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticNormalizedInputSha256: receipt.normalizedInputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }), row(6, "turn.completed", {}), row(7, "run.result.accepted", { result: structuredClone(result) }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1 })]; const expected = { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: readCopilotToolEvidence(rows, "run")[0]!, summary: "EXACT-MARKER" }; return { rows, expected }; @@ -28,14 +32,59 @@ function setField(r: any, key: string, value: string) { frame(r).payload.details describe("Copilot semantic completion public-event oracle", () => { it("versions the actual Copilot suite oracle without changing denial settlement", () => { const suite = runnerSuites.find(s => s.id === "copilot-protection")!; - expect(suite.definitionMetadata).toMatchObject({ version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" }); - expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 6 } })); + expect(suite.definitionMetadata).toMatchObject({ version: 8, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v2", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" }); + expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 7 } })); }); it("joins exact native lifecycle, authoritative callback, proposed content and accepted control-plane result", () => { const { rows, expected } = fixture(); const proof = readCopilotSemanticCompletion(rows, expected); expect(proof.nativeToolCallId).toBe("finish-native"); expect(proof.acceptedSeq).toBe(7); expect(proof.summarySha256).toBe(hash("EXACT-MARKER")); expect(JSON.stringify(proof)).not.toContain("EXACT-MARKER"); }); + it("binds distinct raw and production-normalized input digests without filling defaults in the oracle", () => { + const { rows, expected } = fixture(); + expect(rawInput).not.toHaveProperty("schema"); + expect(rawInput).not.toHaveProperty("artifacts"); + expect(rawInput).not.toHaveProperty("attentionRequests"); + expect(result).toMatchObject({ schema: "paperclip.run_result.v1", artifacts: [], attentionRequests: [] }); + expect(receipt.inputSha256).not.toBe(receipt.normalizedInputSha256); + expect(readCopilotSemanticCompletion(rows, expected)).toMatchObject({ + schema: "paperclip.e2e.copilot-semantic-completion.v2", + inputSha256: receipt.inputSha256, normalizedInputSha256: receipt.normalizedInputSha256, + }); + // Same authenticated raw/native join is necessary but cannot substitute for + // the invocation-captured normalized body digest. + setField(rows[3], "normalizedInputSha256", receipt.inputSha256); + setField(rows[4], "semanticNormalizedInputSha256", receipt.inputSha256); + expect(() => readCopilotSemanticCompletion(rows, expected)).toThrow("canonical accepted result"); + }); + it.each(["missing", "null", "wrong", "uppercase", "malformed"])("rejects %s normalized finish digest", value => { + const f = fixture(); + if (value === "missing") { + frame(f.rows[3]!).payload.details = frame(f.rows[3]!).payload.details.filter((d: any) => d.name !== "normalizedInputSha256"); + frame(f.rows[4]!).payload.details = frame(f.rows[4]!).payload.details.filter((d: any) => d.name !== "semanticNormalizedInputSha256"); + } else { + const digest = value === "null" ? "null" : value === "wrong" ? hash("foreign-normalized-input") : value === "uppercase" ? "A".repeat(64) : "not-a-digest"; + setField(f.rows[3], "normalizedInputSha256", digest); setField(f.rows[4], "semanticNormalizedInputSha256", digest); + } + expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); + }); + it("allows an explicit null native digest as diagnostic data, never as successful finish authority", () => { + const f = fixture(); setField(f.rows[4], "semanticNormalizedInputSha256", "null"); + expect(readCopilotToolEvidence(f.rows, "run").at(-1)?.semanticNormalizedInputSha256).toBeNull(); + expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow("native receipt mismatch"); + }); + it.each(["only", "alongside-v2"])("rejects legacy v1 authority %s as fresh qualification", mode => { + const f = fixture(), old = structuredClone(f.rows[3]!); + frame(old).payload.category = "paperclip_semantic_tool_receipt_v1"; + setField(old, "schema", "paperclip.semantic_tool_receipt.v1"); + frame(old).payload.details = frame(old).payload.details.filter((d: any) => d.name !== "normalizedInputSha256"); + if (mode === "only") f.rows[3] = old; else f.rows.push(old); + expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); + }); + it("does not repair an unnormalized proposed and accepted body by adding defaults", () => { + const f = fixture(); frame(f.rows[2]!).payload = structuredClone(rawInput); frame(f.rows[6]!).payload.result = structuredClone(rawInput); + expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow("canonical accepted result"); + }); it.each([0, 1, 2, 3, 4, 5, 6])("rejects missing required row %s", index => { const f = fixture(); f.rows.splice(index, 1); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); it.each([1, 2, 3, 4, 5, 6])("rejects duplicate required row %s", index => { const f = fixture(); f.rows.push(structuredClone(f.rows[index]!)); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); it.each(["companyId", "runId"])("rejects foreign durable %s", key => { const f = fixture(); (f.rows[3] as any)[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); @@ -44,7 +93,7 @@ describe("Copilot semantic completion public-event oracle", () => { }); it.each([null, undefined])("rejects missing envelope turn %s even with native provenance intact", turn => { const f = fixture(); frame(f.rows[4]!).turnId = turn; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); it.each(["sessionId", "turnId", "method", "eventType"])("rejects authoritative provenance %s mismatch", key => { const f = fixture(); frame(f.rows[3]!).payload.provenance[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); - it.each(["callIdentitySha256", "inputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each(["callIdentitySha256", "inputSha256", "normalizedInputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); it("does not infer acceptance from returned receipt or a tool named paperclip_finish", () => { const f = fixture(); frame(f.rows[6]!).payload.result = { accepted: false, summary: "EXACT-MARKER" }; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); const g = fixture(); frame(g.rows[4]!).payload.title = "paperclip_finish"; frame(g.rows[4]!).payload.details = frame(g.rows[4]!).payload.details.filter((d: any) => !d.name.startsWith("semantic")); expect(() => readCopilotSemanticCompletion(g.rows, g.expected)).toThrow(); @@ -86,7 +135,7 @@ describe("Copilot semantic completion public-event oracle", () => { frame(r).sourceInstanceId = "foreign"; r.sourceInstanceId = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: f.command, summary: "EXACT-MARKER" })).toThrow(); }); - it.each(["missing-pending", "missing-terminal", "duplicate-pending", "duplicate-terminal", "extra-completed-only", "extra-complete-read", "wrong-shell", "wrong-command", "failed", "nonzero", "reordered", "late", "wrong-operation", "mutation", "read-path", "foreign-session", "foreign-turn", "foreign-run", "permission", "semantic-fields"])("rejects invalid shell-read lifecycle: %s", failure => { + it.each(["missing-pending", "missing-terminal", "duplicate-pending", "duplicate-terminal", "extra-completed-only", "extra-complete-read", "wrong-shell", "wrong-command", "failed", "nonzero", "reordered", "late", "wrong-operation", "mutation", "read-path", "foreign-session", "foreign-turn", "foreign-run", "permission", "semantic-fields", "normalized-semantic-field"])("rejects invalid shell-read lifecycle: %s", failure => { const f = attachedFixture(), start = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")!, end = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "completed")!; if (failure === "missing-pending") f.notices.splice(f.notices.indexOf(start), 1); if (failure === "missing-terminal") f.notices.splice(f.notices.indexOf(end), 1); @@ -108,6 +157,7 @@ describe("Copilot semantic completion public-event oracle", () => { if (failure === "foreign-run") start.runId = "foreign"; if (failure === "permission") start.stage = "permission_requested"; if (failure === "semantic-fields") end.semanticCallIdentitySha256 = receipt.callIdentitySha256; + if (failure === "normalized-semantic-field") end.semanticNormalizedInputSha256 = receipt.normalizedInputSha256; expect(onlyCopilotAttachedOperations(f.notices, f.command, f.proof)).toBe(false); }); it("rejects duplicate shell origins and arbitrary extra operations", () => { diff --git a/tests/runner-e2e/copilot-semantic-evidence.ts b/tests/runner-e2e/copilot-semantic-evidence.ts index 6f1439dfd41..029f09c7f93 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.ts @@ -8,11 +8,11 @@ const hash = (v: string) => createHash("sha256").update(v).digest("hex"); const canonical = (v: unknown): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical((v as Row)[k])}`).join(",")}}` : JSON.stringify(v) ?? "undefined"; const digest = (v: unknown) => hash(canonical(v)); -const receiptKeys = ["callIdentitySha256", "inputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(","); +const receiptKeys = ["callIdentitySha256", "inputSha256", "normalizedInputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(","); export interface CopilotSemanticCompletionProof { - schema: "paperclip.e2e.copilot-semantic-completion.v1"; + schema: "paperclip.e2e.copilot-semantic-completion.v2"; runId: string; turnId: string; nativeSessionId: string; normalizedSessionId: string; sourceInstanceId: string; - nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; resultSha256: string; + nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; normalizedInputSha256: string; resultSha256: string; proposedSeq: number; receiptSeq: number; nativeCompletedSeq: number; turnCompletedSeq: number; acceptedSeq: number; summarySha256: string; } @@ -45,26 +45,26 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected frame(rs[0]!, "runner"); } if (expected.command.runId !== expected.runId || expected.command.turnId !== expected.turnId || expected.command.sessionId !== expected.nativeSessionId) fail("foreign command"); - const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1"); + const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v2" || rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1"); // This authored case asks for one finish, not arbitrary semantic actions. if (authorityRows.length !== 1) fail("expected one authoritative finish receipt"); const authorityRow = authorityRows[0]!, authority = frame(authorityRow, "runner"), payload = rec(authority.payload), origin = rec(payload.provenance); - if (payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded" + if (payload.category !== "paperclip_semantic_tool_receipt_v2" || payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded" || origin.method !== "paperclip/semantic_tool_result" || origin.eventType !== "semantic_result" - || origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 7) fail("invalid receipt provenance"); + || origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 8) fail("invalid receipt provenance"); const fields: Record = {}; for (const d of payload.details) { if (Object.keys(rec(d)).sort().join(",") !== "name,value" || typeof d.name !== "string" || typeof d.value !== "string" || d.value.length > 256 || Object.hasOwn(fields, d.name)) fail("malformed receipt detail"); fields[d.name] = d.value; } - if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v1" + if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v2" || fields.operationId !== "paperclip_finish" || fields.outcome !== "returned" - || ![fields.callIdentitySha256, fields.inputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt"); + || ![fields.callIdentitySha256, fields.inputSha256, fields.normalizedInputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt"); const matches = notices.filter(n => n.semanticCallIdentitySha256 === fields.callIdentitySha256); if (matches.length !== 1) fail("missing or duplicate native match"); const native = matches[0]!; if (native.stage !== "tool" || native.status !== "completed" || native.sessionId !== expected.nativeSessionId || native.turnId !== expected.turnId - || native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch"); + || native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticNormalizedInputSha256 !== fields.normalizedInputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch"); const group = notices.filter(n => n.toolCallId === native.toolCallId); const pending = group.filter(n => n.stage === "tool" && n.status === "pending"); if (pending.length !== 1 || group.filter(n => n.status === "completed" || n.status === "failed").length !== 1 @@ -82,8 +82,11 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected // Native correlation uses the invocation-captured callback call hash. The // canonical proposal itemId may instead identify the run: never interpret it // as a bridge call ID. Unique same-turn result bodies supply the independent - // acceptance proof, joined to that exact invocation's input digest. - if (digest(result) !== fields.inputSha256 + // acceptance proof, joined to the normalized digest captured by the same + // invocation after validation. Raw provider arguments may omit defaults; + // their separate digest remains joined to the native receipt above. Never + // reconstruct or normalize a proposal here to make its digest match. + if (digest(result) !== fields.normalizedInputSha256 || result.schema !== "paperclip.run_result.v1" || result.reportedWorkDisposition !== "done" || result.summary !== expected.summary || canonical(result) !== canonical(acceptedResult)) fail("canonical accepted result does not match exact finish"); const pendingFrame = frame(all.find(r => r.seq === pending[0]!.seq)!, "runner"); @@ -92,9 +95,9 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected && authorityRow.seq < native.seq && native.seq < terminals[0]!.seq && terminals[0]!.seq < acceptedRows[0]!.seq) || !(base.sourceSeq < pendingFrame.sourceSeq && pendingFrame.sourceSeq < proposed.sourceSeq && proposed.sourceSeq < authority.sourceSeq && authority.sourceSeq < completedFrame.sourceSeq && completedFrame.sourceSeq < terminal.sourceSeq)) fail("invalid causal order"); - return { schema: "paperclip.e2e.copilot-semantic-completion.v1", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId, + return { schema: "paperclip.e2e.copilot-semantic-completion.v2", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId, normalizedSessionId: base.normalizedSessionId, sourceInstanceId: base.sourceInstanceId, nativeToolCallId: native.toolCallId, - callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, resultSha256: fields.resultSha256!, + callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, normalizedInputSha256: fields.normalizedInputSha256!, resultSha256: fields.resultSha256!, proposedSeq: proposedRows[0]!.seq, receiptSeq: authorityRow.seq, nativeCompletedSeq: native.seq, turnCompletedSeq: terminals[0]!.seq, acceptedSeq: acceptedRows[0]!.seq, summarySha256: hash(expected.summary) }; } @@ -117,7 +120,7 @@ export function onlyCopilotAttachedOperations(notices: readonly CopilotToolNotic : i === group.length - 1 ? n.status === "completed" : n.status === "in_progress")); }; const semanticFree = (n: CopilotToolNotice) => n.semanticOperationId === undefined && n.semanticCallIdentitySha256 === undefined - && n.semanticInputSha256 === undefined && n.semanticResultSha256 === undefined && n.semanticOutcome === undefined; + && n.semanticInputSha256 === undefined && n.semanticNormalizedInputSha256 === undefined && n.semanticResultSha256 === undefined && n.semanticOutcome === undefined; const commandGroup = groups.get(command.toolCallId) ?? []; if (!complete(commandGroup) || commandGroup[0]!.seq !== command.seq) return false; const started = commandGroup.at(-1)!;