From 4a876efef5202f63c8006d86e47c8fb12d02d6bb Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 22:00:11 -0500 Subject: [PATCH] test(runner-e2e): require correlated accepted Copilot completion Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 18 ++++ tests/runner-e2e/README.md | 18 ++++ tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/copilot-evidence.ts | 10 +- .../copilot-protection-flow.test.ts | 2 +- tests/runner-e2e/copilot-protection-flow.ts | 9 +- .../copilot-semantic-evidence.test.ts | 77 +++++++++++++++ tests/runner-e2e/copilot-semantic-evidence.ts | 96 +++++++++++++++++++ 8 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/copilot-semantic-evidence.test.ts create mode 100644 tests/runner-e2e/copilot-semantic-evidence.ts diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 139c3bd4791..29a0316692f 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -455,3 +455,21 @@ The existing `copilot-protection` denial remains distinct: rejecting a permissio before Stop does not exercise this pending-callback boundary. This new suite has pure calibration and wiring tests, not a paid qualification result. Provider process death is not simulated by substituting the chat runner-worker crash hook. + +### Copilot attached semantic completion evidence (suite 7) + +Attached settlement now requires one native `paperclip_finish` lifecycle joined +to an invocation-captured, bounded Paperclip bridge receipt by exact call, input +and result digests in the same run/turn/native session and durable source stream. +A matching display title is not authority. The exact proposed completion input +must match a control-plane `run.result.accepted` body and the requested summary. +Transport `returned` alone, including a returned rejection, cannot pass. The +visible exact terminal comment remains an independent assertion. Both local and +Daytona cases reject extra native operations; existing attested bootstrap reads, +command settlement, process retirement and marker checks remain required. + +The receipt reader fails closed when the production native receipt projection is +missing, partial, duplicated or outside its bounds (including 256 KiB for the +whole native rawOutput, which may repeat text). This oracle depends on the new +production semantic-receipt contract and does not regrade earlier failed runs. +Denial case version 5 and denial settlement schema v3 are unchanged. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index ee1d7c0ac05..47ea91a3964 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1503,3 +1503,21 @@ older denial or cancellation results. Current live qualification is pending. Provider death during a pending callback remains a separate uncovered case: the existing chat worker-crash hook targets a runner worker and cannot establish safe ownership of the native provider process in both environments. + +### Copilot attached semantic completion evidence (suite 7) + +Attached settlement now requires one native `paperclip_finish` lifecycle joined +to an invocation-captured, bounded Paperclip bridge receipt by exact call, input +and result digests in the same run/turn/native session and durable source stream. +A matching display title is not authority. The exact proposed completion input +must match a control-plane `run.result.accepted` body and the requested summary. +Transport `returned` alone, including a returned rejection, cannot pass. The +visible exact terminal comment remains an independent assertion. Both local and +Daytona cases reject extra native operations; existing attested bootstrap reads, +command settlement, process retirement and marker checks remain required. + +The receipt reader fails closed when the production native receipt projection is +missing, partial, duplicated or outside its bounds (including 256 KiB for the +whole native rawOutput, which may repeat text). This oracle depends on the new +production semantic-receipt contract and does not regrade earlier failed runs. +Denial case version 5 and denial settlement schema v3 are unchanged. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 74df4128937..427a057a50e 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1059,7 +1059,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 6, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-evidence.ts b/tests/runner-e2e/copilot-evidence.ts index 6a74c589d53..d8272d96242 100644 --- a/tests/runner-e2e/copilot-evidence.ts +++ b/tests/runner-e2e/copilot-evidence.ts @@ -2,6 +2,7 @@ export interface CopilotToolNotice { runId: string; sessionId: string; turnId: string; toolCallId: string; observedAtMs: number; seq: number; + semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticResultSha256?: string; semanticOutcome?: "returned" | "error"; stage: "tool" | "permission_requested" | "permission_delivered"; status?: "pending" | "in_progress" | "completed" | "failed"; operation?: "edit" | "execute" | "read"; target?: string; requestId?: string; @@ -12,7 +13,7 @@ export interface CopilotToolNotice { const rec = (v: unknown): Record => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]"); const enums = { stage: ["tool", "permission_requested", "permission_delivered"], status: ["pending", "in_progress", "completed", "failed"], operation: ["edit", "execute", "read"], outcome: ["allow_once", "allow_always", "reject_once", "cancel"], mode: ["sync", "async"], shellState: ["started", "completed"] }; -const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode"]); +const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]); export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: string): CopilotToolNotice[] { const result: CopilotToolNotice[] = []; for (const row of rows) { @@ -42,6 +43,13 @@ export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: if (fields.commandSha256 !== undefined) { if (!/^sha256:[a-f0-9]{64}$/u.test(fields.commandSha256)) throw new Error("Invalid command digest"); notice.commandSha256 = fields.commandSha256; } if (fields.shellId !== undefined) { if (!/^[A-Za-z0-9_.-]{1,80}$/u.test(fields.shellId)) throw new Error("Invalid shell identity"); notice.shellId = fields.shellId; } if (fields.exitCode !== undefined) { if (!/^-?\d{1,10}$/u.test(fields.exitCode) || !Number.isSafeInteger(Number(fields.exitCode))) throw new Error("Invalid exit code"); notice.exitCode = Number(fields.exitCode); } + const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]; + if (semanticKeys.some(k => fields[k] !== undefined)) { + if (!semanticKeys.every(k => fields[k] !== undefined) || fields.stage !== "tool" || !["completed", "failed"].includes(fields.status!) + || !/^[A-Za-z0-9_.:-]{1,256}$/.test(fields.semanticOperationId!) || !["returned", "error"].includes(fields.semanticOutcome!) + || ["semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256"].some(k => !/^[a-f0-9]{64}$/.test(fields[k]!))) throw new Error("Invalid semantic receipt fields"); + for (const key of semanticKeys) notice[key] = fields[key]; + } result.push(notice as unknown as CopilotToolNotice); } return result; diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 5748d9630f8..8c286a5917e 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 6, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 7, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index eb2b8902463..659bde01dba 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -4,6 +4,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; +import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; @@ -239,9 +240,11 @@ export async function runCopilotProtectionFlow(input: { markerMatches: localMarkerMatches, afterCleanupMarkerMatches: localAfterCleanupMarkerMatches }); const call = matched?.call; check("single-exact-command", Boolean(matched), "Exactly one native execution matches the fixture command with at most eight leading ASCII SPACE/TAB bytes"); - if (remote) check("no-extra-native-operation", copilotActionNotices(notices, call!, { actionFile: remoteFixture!.actionFile, events: runEvents }).every(n => - n.runId === call!.runId && n.sessionId === call!.sessionId && n.turnId === call!.turnId - && (n.toolCallId === call!.toolCallId || n.commandToolCallId === call!.toolCallId)), "Only setup reads and the exact native attached command/result are allowed"); + const semantic = readCopilotSemanticCompletion(runEvents, { companyId: fixtures.company.id, runId: call!.runId, turnId: call!.turnId, + nativeSessionId: call!.sessionId, command: call!, summary: execution.task.buildVisibleMarker(nonce) }); + await input.evidence("copilot-semantic-completion.json", semantic); + check("accepted-canonical-completion", true, "One exact native finish receipt matches the proposed and control-plane accepted result; transport success alone is insufficient"); + check("no-extra-native-operation", onlyCopilotAttachedOperations(copilotActionNotices(notices, call!, remoteFixture ? { actionFile: remoteFixture.actionFile, events: runEvents } : undefined), call!, semantic), "Only attested setup reads, the exact attached command/result, and one authoritatively correlated accepted finish are allowed"); const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started"); const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed"); const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent; diff --git a/tests/runner-e2e/copilot-semantic-evidence.test.ts b/tests/runner-e2e/copilot-semantic-evidence.test.ts new file mode 100644 index 00000000000..e142916daf2 --- /dev/null +++ b/tests/runner-e2e/copilot-semantic-evidence.test.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; +import { readCopilotToolEvidence } from "./copilot-evidence.js"; +import { runnerSuites, suiteDefinitionHash } from "./catalog.js"; +const hash = (v: string) => createHash("sha256").update(v).digest("hex"); +const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v); +const result = { schema: "paperclip.run_result.v1", summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], artifacts: [], verification: [], attentionRequests: [] }; +const receipt = { schema: "paperclip.semantic_tool_receipt.v1", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" }; +function row(seq: number, eventType: string, payload: any, overrides: any = {}) { + const e = { schema: "paperclip.prp.event.v1", eventType, runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "source", sourceKind: "runner", sourceSeq: seq, emittedAt: "2026-09-30T12:00:00Z", payload, ...overrides }; + return { companyId: "company", runId: "run", seq, eventType, sourceInstanceId: e.sourceInstanceId, sourceSeq: e.sourceSeq, payload: { prpEvent: e } }; +} +function notice(seq: number, toolCallId: string, status: string, fields: any = {}) { + return row(seq, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "copilot_tool_evidence_v1", scope: "turn", provenance: { method: "session/update", eventType: "tool", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "tool", toolCallId, status, ...fields }).map(([name, value]) => ({ name, value: String(value) })) }); +} +function fixture() { + const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "3" }), + row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v1", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }), + notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }), + row(6, "turn.completed", {}), row(7, "run.result.accepted", { result: structuredClone(result) }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1 })]; + const expected = { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: readCopilotToolEvidence(rows, "run")[0]!, summary: "EXACT-MARKER" }; + return { rows, expected }; +} +const frame = (r: any) => r.payload.prpEvent; +function setField(r: any, key: string, value: string) { frame(r).payload.details.find((d: any) => d.name === key).value = value; } +describe("Copilot semantic completion public-event oracle", () => { + it("versions the actual Copilot suite oracle without changing denial settlement", () => { + const suite = runnerSuites.find(s => s.id === "copilot-protection")!; + expect(suite.definitionMetadata).toMatchObject({ version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" }); + expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 6 } })); + }); + it("joins exact native lifecycle, authoritative callback, proposed content and accepted control-plane result", () => { + const { rows, expected } = fixture(); const proof = readCopilotSemanticCompletion(rows, expected); + expect(proof.nativeToolCallId).toBe("finish-native"); expect(proof.acceptedSeq).toBe(7); expect(proof.summarySha256).toBe(hash("EXACT-MARKER")); + expect(JSON.stringify(proof)).not.toContain("EXACT-MARKER"); + }); + it.each([0, 1, 2, 3, 4, 5, 6])("rejects missing required row %s", index => { const f = fixture(); f.rows.splice(index, 1); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each([1, 2, 3, 4, 5, 6])("rejects duplicate required row %s", index => { const f = fixture(); f.rows.push(structuredClone(f.rows[index]!)); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each(["companyId", "runId"])("rejects foreign durable %s", key => { const f = fixture(); (f.rows[3] as any)[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each(["turnId", "normalizedSessionId", "sourceInstanceId", "sourceKind", "eventType", "schema"])("rejects foreign or malformed envelope %s", key => { + for (const index of [1, 2, 3, 4, 5, 6]) { const f = fixture(); (frame(f.rows[index]!))[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } + }); + it.each([null, undefined])("rejects missing envelope turn %s even with native provenance intact", turn => { const f = fixture(); frame(f.rows[4]!).turnId = turn; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each(["sessionId", "turnId", "method", "eventType"])("rejects authoritative provenance %s mismatch", key => { const f = fixture(); frame(f.rows[3]!).payload.provenance[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it.each(["callIdentitySha256", "inputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it("does not infer acceptance from returned receipt or a tool named paperclip_finish", () => { + const f = fixture(); frame(f.rows[6]!).payload.result = { accepted: false, summary: "EXACT-MARKER" }; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); + const g = fixture(); frame(g.rows[4]!).payload.title = "paperclip_finish"; frame(g.rows[4]!).payload.details = frame(g.rows[4]!).payload.details.filter((d: any) => !d.name.startsWith("semantic")); expect(() => readCopilotSemanticCompletion(g.rows, g.expected)).toThrow(); + }); + it.each(["failed", "cancelled", "interrupted"])("rejects %s terminal", status => { const f = fixture(); f.rows[5]!.eventType = `turn.${status}`; frame(f.rows[5]!).eventType = `turn.${status}`; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }); + it("rejects changed accepted body, forged call ID, wrong summary and rejected result", () => { + for (const mutate of [(f: ReturnType) => { frame(f.rows[6]!).payload.result.completionClaim.remainingWork.push("unfinished"); }, (f: ReturnType) => { frame(f.rows[2]!).itemId = "4"; }, (f: ReturnType) => { f.expected.summary = "other"; }, (f: ReturnType) => { f.rows.push(row(8, "run.result.rejected", {})); }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } + }); + it("rejects duplicate/unknown/oversized receipt details and partial native fields", () => { + for (const mutate of [(r: any) => frame(r).payload.details.push({ name: "secret", value: "untrusted" }), (r: any) => frame(r).payload.details.push(frame(r).payload.details[0]), (r: any) => setField(r, "operationId", "x".repeat(300))]) { const f = fixture(); mutate(f.rows[3]); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } + const f = fixture(); frame(f.rows[4]!).payload.details.pop(); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); + }); + it("rejects mutation metadata, extra lifecycle, reordered receipt and source sequence drift", () => { + for (const mutate of [(f: ReturnType) => frame(f.rows[1]!).payload.details.push({ name: "operation", value: "edit" }), (f: ReturnType) => f.rows.push(notice(8, "finish-native", "pending")), (f: ReturnType) => { f.rows[3]!.seq = 8; }, (f: ReturnType) => { frame(f.rows[3]!).sourceSeq = 20; f.rows[3]!.sourceSeq = 20; }, (f: ReturnType) => { frame(f.rows[1]!).sourceSeq = 20; f.rows[1]!.sourceSeq = 20; }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); } + }); + it("allows only exact command and correlated finish; unknown reads/edits/delegation remain failures", () => { + const f = fixture(), proof = readCopilotSemanticCompletion(f.rows, f.expected), notices = readCopilotToolEvidence(f.rows, "run"); + expect(onlyCopilotAttachedOperations(notices, f.expected.command, proof)).toBe(true); + for (const operation of [undefined, "read", "edit", "execute"] as const) expect(onlyCopilotAttachedOperations([...notices, { ...f.expected.command, toolCallId: "extra", operation }], f.expected.command, proof)).toBe(false); + expect(onlyCopilotAttachedOperations(notices, f.expected.command, { ...proof, nativeToolCallId: "command" })).toBe(false); + expect(onlyCopilotAttachedOperations([{ ...notices[1]!, sessionId: "foreign" }], f.expected.command, proof)).toBe(false); + }); + it("keeps actual flow correlation before no-extra-operation gate and keeps visible marker independent", async () => { + const source = await readFile(new URL("./copilot-protection-flow.ts", import.meta.url), "utf8"); + expect(source.indexOf("readCopilotSemanticCompletion(runEvents")).toBeLessThan(source.indexOf('check("no-extra-native-operation", onlyCopilotAttachedOperations(')); + expect(source).toContain("undefined), call!, semantic)"); + expect(source).toContain('check("exact-completion-marker", comments.filter'); + expect(source).not.toContain('if (remote) check("no-extra-native-operation"'); + }); +}); diff --git a/tests/runner-e2e/copilot-semantic-evidence.ts b/tests/runner-e2e/copilot-semantic-evidence.ts new file mode 100644 index 00000000000..1f0a7935d7a --- /dev/null +++ b/tests/runner-e2e/copilot-semantic-evidence.ts @@ -0,0 +1,96 @@ +import { createHash } from "node:crypto"; +import { readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; + +type Row = Record; +const rec = (v: unknown): Row => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Row : {}; +const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]"); +const hash = (v: string) => createHash("sha256").update(v).digest("hex"); +const canonical = (v: unknown): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" + ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical((v as Row)[k])}`).join(",")}}` : JSON.stringify(v) ?? "undefined"; +const digest = (v: unknown) => hash(canonical(v)); +const receiptKeys = ["callIdentitySha256", "inputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(","); +export interface CopilotSemanticCompletionProof { + schema: "paperclip.e2e.copilot-semantic-completion.v1"; + runId: string; turnId: string; nativeSessionId: string; normalizedSessionId: string; sourceInstanceId: string; + nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; resultSha256: string; + proposedSeq: number; receiptSeq: number; nativeCompletedSeq: number; turnCompletedSeq: number; acceptedSeq: number; + summarySha256: string; +} +/** Independent public-event oracle. A transport receipt alone never proves completion acceptance. */ +export function readCopilotSemanticCompletion(rows: readonly unknown[], expected: { + companyId: string; runId: string; turnId: string; nativeSessionId: string; command: CopilotToolNotice; summary: string; +}): CopilotSemanticCompletionProof { + const fail = (message: string): never => { throw new Error(`Copilot semantic completion: ${message}`); }; + const all = rows.map(rec), notices = readCopilotToolEvidence(rows, expected.runId); + const commandRows = all.filter(r => r.seq === expected.command.seq); + if (commandRows.length !== 1) fail("missing exact command origin"); + const base = rec(rec(commandRows[0]!.payload).prpEvent); + if (!id(base.normalizedSessionId) || !id(base.sourceInstanceId)) fail("missing command stream"); + function frame(r: Row, sourceKind: "runner" | "control_plane") { + const e = rec(rec(r.payload).prpEvent); + if (r.companyId !== expected.companyId || r.runId !== expected.runId || !Number.isSafeInteger(r.seq) || r.seq < 1 + || e.schema !== "paperclip.prp.event.v1" || e.eventType !== r.eventType || e.sourceKind !== sourceKind + || e.runId !== expected.runId || e.turnId !== expected.turnId || e.normalizedSessionId !== base.normalizedSessionId + || !id(e.sourceInstanceId) || r.sourceInstanceId !== e.sourceInstanceId || r.sourceSeq !== e.sourceSeq + || !Number.isSafeInteger(e.sourceSeq) || e.sourceSeq < 1 + || (sourceKind === "runner" && e.sourceInstanceId !== base.sourceInstanceId)) fail("foreign or malformed durable frame"); + return e; + } + frame(commandRows[0]!, "runner"); + if (expected.command.runId !== expected.runId || expected.command.turnId !== expected.turnId || expected.command.sessionId !== expected.nativeSessionId) fail("foreign command"); + const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1"); + // This authored case asks for one finish, not arbitrary semantic actions. + if (authorityRows.length !== 1) fail("expected one authoritative finish receipt"); + const authorityRow = authorityRows[0]!, authority = frame(authorityRow, "runner"), payload = rec(authority.payload), origin = rec(payload.provenance); + if (payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded" + || origin.method !== "paperclip/semantic_tool_result" || origin.eventType !== "semantic_result" + || origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 7) fail("invalid receipt provenance"); + const fields: Record = {}; + for (const d of payload.details) { + if (Object.keys(rec(d)).sort().join(",") !== "name,value" || typeof d.name !== "string" || typeof d.value !== "string" || d.value.length > 256 || Object.hasOwn(fields, d.name)) fail("malformed receipt detail"); + fields[d.name] = d.value; + } + if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v1" + || fields.operationId !== "paperclip_finish" || fields.outcome !== "returned" + || ![fields.callIdentitySha256, fields.inputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt"); + const matches = notices.filter(n => n.semanticCallIdentitySha256 === fields.callIdentitySha256); + if (matches.length !== 1) fail("missing or duplicate native match"); + const native = matches[0]!; + if (native.stage !== "tool" || native.status !== "completed" || native.sessionId !== expected.nativeSessionId || native.turnId !== expected.turnId + || native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch"); + const group = notices.filter(n => n.toolCallId === native.toolCallId); + const pending = group.filter(n => n.stage === "tool" && n.status === "pending"); + if (pending.length !== 1 || group.filter(n => n.status === "completed" || n.status === "failed").length !== 1 + || group.some(n => n.stage !== "tool" || n.sessionId !== expected.nativeSessionId || n.turnId !== expected.turnId + || n.operation !== undefined || n.target !== undefined || n.commandSha256 !== undefined || n.shellId !== undefined || n.commandToolCallId !== undefined + || !["pending", "in_progress", "completed"].includes(n.status ?? "") || n.seq < pending[0]!.seq || n.seq > native.seq)) fail("not one semantic-only native lifecycle"); + for (const n of group) { + const rs = all.filter(r => r.seq === n.seq); if (rs.length !== 1) fail("duplicate native row"); frame(rs[0]!, "runner"); + } + const proposedRows = all.filter(r => r.eventType === "run.result.proposed"), acceptedRows = all.filter(r => r.eventType === "run.result.accepted"), terminals = all.filter(r => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(r.eventType) && rec(rec(r.payload).prpEvent).turnId === expected.turnId); + if (proposedRows.length !== 1 || acceptedRows.length !== 1 || terminals.length !== 1 || terminals[0]!.eventType !== "turn.completed" + || all.some(r => r.eventType === "run.result.rejected")) fail("missing, rejected or ambiguous completion"); + const proposed = frame(proposedRows[0]!, "runner"), accepted = frame(acceptedRows[0]!, "control_plane"), terminal = frame(terminals[0]!, "runner"); + const result = rec(proposed.payload), acceptedResult = rec(rec(accepted.payload).result); + if (!id(proposed.itemId) || hash(proposed.itemId) !== fields.callIdentitySha256 || digest(result) !== fields.inputSha256 + || result.schema !== "paperclip.run_result.v1" || result.reportedWorkDisposition !== "done" || result.summary !== expected.summary + || canonical(result) !== canonical(acceptedResult)) fail("canonical accepted result does not match exact finish"); + const pendingFrame = frame(all.find(r => r.seq === pending[0]!.seq)!, "runner"); + const completedFrame = frame(all.find(r => r.seq === native.seq)!, "runner"); + if (!(expected.command.seq < pending[0]!.seq && pending[0]!.seq < proposedRows[0]!.seq && proposedRows[0]!.seq < authorityRow.seq + && authorityRow.seq < native.seq && native.seq < terminals[0]!.seq && terminals[0]!.seq < acceptedRows[0]!.seq) + || !(base.sourceSeq < pendingFrame.sourceSeq && pendingFrame.sourceSeq < proposed.sourceSeq && proposed.sourceSeq < authority.sourceSeq + && authority.sourceSeq < completedFrame.sourceSeq && completedFrame.sourceSeq < terminal.sourceSeq)) fail("invalid causal order"); + return { schema: "paperclip.e2e.copilot-semantic-completion.v1", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId, + normalizedSessionId: base.normalizedSessionId, sourceInstanceId: base.sourceInstanceId, nativeToolCallId: native.toolCallId, + callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, resultSha256: fields.resultSha256!, + proposedSeq: proposedRows[0]!.seq, receiptSeq: authorityRow.seq, nativeCompletedSeq: native.seq, turnCompletedSeq: terminals[0]!.seq, acceptedSeq: acceptedRows[0]!.seq, summarySha256: hash(expected.summary) }; +} + +/** Called only after the completion proof and bootstrap-read oracle succeed. */ +export function onlyCopilotAttachedOperations(notices: readonly CopilotToolNotice[], command: CopilotToolNotice, proof: CopilotSemanticCompletionProof): boolean { + return proof.runId === command.runId && proof.turnId === command.turnId && proof.nativeSessionId === command.sessionId + && proof.nativeToolCallId !== command.toolCallId && notices.every(n => + n.runId === command.runId && n.sessionId === command.sessionId && n.turnId === command.turnId + && (n.toolCallId === command.toolCallId || n.commandToolCallId === command.toolCallId || n.toolCallId === proof.nativeToolCallId)); +}