From 3f552d753a78b7b546d23d4f85295b28eb4cc3b2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 17:15:46 -0500 Subject: [PATCH] test: count all Copilot origins and verify marker after cleanup Co-Authored-By: Paperclip --- .../copilot-protection-evidence.test.ts | 24 +++++++++++++++++++ .../runner-e2e/copilot-protection-evidence.ts | 14 +++++++++++ tests/runner-e2e/copilot-protection-flow.ts | 12 ++++++---- 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/copilot-protection-evidence.test.ts create mode 100644 tests/runner-e2e/copilot-protection-evidence.ts diff --git a/tests/runner-e2e/copilot-protection-evidence.test.ts b/tests/runner-e2e/copilot-protection-evidence.test.ts new file mode 100644 index 00000000000..70bef038c04 --- /dev/null +++ b/tests/runner-e2e/copilot-protection-evidence.test.ts @@ -0,0 +1,24 @@ +import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; +import type { CopilotToolNotice } from "./copilot-evidence.js"; +const notice: CopilotToolNotice = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "edit", stage: "tool", operation: "edit", observedAtMs: 1, seq: 1 }; +describe("Copilot protection independent evidence", () => { + it.each(["in_progress", "completed", "failed"] as const)("counts an alternate origin first seen as %s", status => { + const rows = [{ ...notice, status: "pending" as const }, { ...notice, status }, { ...notice, toolCallId: "alternate", status }]; + expect(countCopilotToolOrigins(rows)).toBe(2); + expect(countCopilotToolOrigins([{ ...notice, status }, { ...notice, sessionId: "other", status }])).toBe(2); + }); + it.each(["unchanged", "changed", "deleted"])("independently reads the marker after %s cleanup", async mode => { + const root = await mkdtemp("/tmp/pc-copilot-marker-"); const path = join(root, "marker"); + try { + await writeFile(path, "expected"); expect(await readFile(path, "utf8")).toBe("expected"); + const matches = await readCopilotMarkerAfterCleanup(async () => { + if (mode === "changed") await writeFile(path, "changed"); + if (mode === "deleted") await unlink(path); + }, path, "expected"); + expect(matches).toBe(mode === "unchanged"); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts new file mode 100644 index 00000000000..01759b0e40a --- /dev/null +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -0,0 +1,14 @@ +import { readFile } from "node:fs/promises"; +import type { CopilotToolNotice } from "./copilot-evidence.js"; + +/** A tool origin exists regardless of the first status emitted by the provider. */ +export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): number { + return new Set(notices.filter(n => n.stage === "tool").map(n => JSON.stringify([n.runId, n.sessionId, n.turnId, n.toolCallId]))).size; +} + +/** Re-read independently after cleanup; a pre-cleanup value is not evidence. */ +export async function readCopilotMarkerAfterCleanup(close: () => Promise, path: string, expected: string): Promise { + await close(); + try { return await readFile(path, "utf8") === expected; } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } +} diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index b399d4d1ffa..f3f0bac023f 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,6 +7,7 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; +import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; @@ -94,16 +95,16 @@ export async function runCopilotProtectionFlow(input: { terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status }, cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope }, cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt, - nativeAttemptsForTarget: new Set(notices.filter(n => n.stage === "tool" && n.operation === "edit" && n.target === target && n.status === "pending").map(n => n.toolCallId)).size, + nativeAttemptsForTarget: countCopilotToolOrigins(notices.filter(n => n.operation === "edit" && n.target === target)), }; await input.evidence("copilot-denial-proof.json", { evidence, processes, notices }); const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup"); check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done"); - check("no-extra-native-operation", new Set(notices.filter(n => n.stage === "tool" && n.status === "pending").map(n => n.toolCallId)).size === 1, "No alternate native edit, command or delegated operation is permitted"); + check("no-extra-native-operation", countCopilotToolOrigins(notices) === 1, "No alternate native edit, command or delegated operation is permitted"); } else { await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && s.processes.captured && s.processes.live.length === 0); const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256); - check("single-exact-command", Boolean(call) && new Set(notices.filter(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256).map(n => n.toolCallId)).size === 1, "Exactly one native execution contains the supplied command digest"); + check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === command!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest"); const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started"); const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed"); const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent; @@ -112,12 +113,13 @@ export async function runCopilotProtectionFlow(input: { const markerMatches = await readFile(markerPath, "utf8") === command!.marker; check("native-client-before-terminal", Boolean(terminal) && external.clientExitedAtMs !== null && external.clientExitedAtMs < Date.parse(terminal.emittedAt), "Independent PID/start observation confirms native client retirement before turn completion"); check("marker-before-terminal", Boolean(terminal) && external.markerWrittenAtMs !== null && external.markerWrittenAtMs < Date.parse(terminal.emittedAt), "The independent fixture wrote its undisclosed marker before turn completion"); + const afterCleanupMarkerMatches = await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker); const grade = gradeCopilotAttachedSettlement({ expected: copilotOrigin(call!), nativeCall: call ? { ...call, operation: call.operation!, mode: call.mode!, detach: call.detach!, commandSha256: call.commandSha256! } : null, expectedCommandSha256: command!.commandSha256, commandExit: external.commandExit, expectedShellId: started?.shellId ?? "", nativeShellResult: result ? { ...result, shellId: result.shellId!, commandToolCallId: result.commandToolCallId!, status: result.status!, exitCode: result.exitCode! } : null, terminal: terminal ? { observedAtMs: Date.parse(terminal.emittedAt), runId: terminal.runId, turnId: terminal.turnId, status: "succeeded" } : null, - cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches: markerMatches }); - await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches }); + cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches }); + await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches, afterCleanupMarkerMatches }); check("attached-settlement-before-terminal", grade.passed, grade.failures.join(", ") || "Owned finite process and native shell settled before the actual turn terminal"); } await load(); check("one-native-run", runs.length === 1 && runs[0]!.runtimeMode === "native", "Exactly one native run was accounted");