test: count all Copilot origins and verify marker after cleanup

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/copilot-protection-evidence.test.ts

@@ -0,0 +1,24 @@
+import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
+import type { CopilotToolNotice } from "./copilot-evidence.js";
+const notice: CopilotToolNotice = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "edit", stage: "tool", operation: "edit", observedAtMs: 1, seq: 1 };
+describe("Copilot protection independent evidence", () => {
+  it.each(["in_progress", "completed", "failed"] as const)("counts an alternate origin first seen as %s", status => {
+    const rows = [{ ...notice, status: "pending" as const }, { ...notice, status }, { ...notice, toolCallId: "alternate", status }];
+    expect(countCopilotToolOrigins(rows)).toBe(2);
+    expect(countCopilotToolOrigins([{ ...notice, status }, { ...notice, sessionId: "other", status }])).toBe(2);
+  });
+  it.each(["unchanged", "changed", "deleted"])("independently reads the marker after %s cleanup", async mode => {
+    const root = await mkdtemp("/tmp/pc-copilot-marker-"); const path = join(root, "marker");
+    try {
+      await writeFile(path, "expected"); expect(await readFile(path, "utf8")).toBe("expected");
+      const matches = await readCopilotMarkerAfterCleanup(async () => {
+        if (mode === "changed") await writeFile(path, "changed");
+        if (mode === "deleted") await unlink(path);
+      }, path, "expected");
+      expect(matches).toBe(mode === "unchanged");
+    } finally { await rm(root, { recursive: true, force: true }); }
+  });
+});

tests/runner-e2e/copilot-protection-evidence.ts

@@ -0,0 +1,14 @@
+import { readFile } from "node:fs/promises";
+import type { CopilotToolNotice } from "./copilot-evidence.js";
+
+/** A tool origin exists regardless of the first status emitted by the provider. */
+export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): number {
+  return new Set(notices.filter(n => n.stage === "tool").map(n => JSON.stringify([n.runId, n.sessionId, n.turnId, n.toolCallId]))).size;
+}
+
+/** Re-read independently after cleanup; a pre-cleanup value is not evidence. */
+export async function readCopilotMarkerAfterCleanup(close: () => Promise<void>, path: string, expected: string): Promise<boolean> {
+  await close();
+  try { return await readFile(path, "utf8") === expected; }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; }
+}

tests/runner-e2e/copilot-protection-flow.ts

@@ -7,6 +7,7 @@ import { createTaskThroughUi } from "./user-actions.js";
 import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
 import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
 import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
+import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
 import type { LiveFixtureValues } from "./live-fixtures.js";
 import type { MatrixExecution } from "./types.js";
 type Row = Record<string, any>;
@@ -94,16 +95,16 @@ export async function runCopilotProtectionFlow(input: {
         terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status },
         cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope },
         cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt,
-        nativeAttemptsForTarget: new Set(notices.filter(n => n.stage === "tool" && n.operation === "edit" && n.target === target && n.status === "pending").map(n => n.toolCallId)).size,
+        nativeAttemptsForTarget: countCopilotToolOrigins(notices.filter(n => n.operation === "edit" && n.target === target)),
       };
       await input.evidence("copilot-denial-proof.json", { evidence, processes, notices });
       const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup");
       check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done");
-      check("no-extra-native-operation", new Set(notices.filter(n => n.stage === "tool" && n.status === "pending").map(n => n.toolCallId)).size === 1, "No alternate native edit, command or delegated operation is permitted");
+      check("no-extra-native-operation", countCopilotToolOrigins(notices) === 1, "No alternate native edit, command or delegated operation is permitted");
     } else {
       await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && s.processes.captured && s.processes.live.length === 0);
       const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256);
-      check("single-exact-command", Boolean(call) && new Set(notices.filter(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256).map(n => n.toolCallId)).size === 1, "Exactly one native execution contains the supplied command digest");
+      check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === command!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest");
       const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started");
       const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed");
       const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent;
@@ -112,12 +113,13 @@ export async function runCopilotProtectionFlow(input: {
       const markerMatches = await readFile(markerPath, "utf8") === command!.marker;
       check("native-client-before-terminal", Boolean(terminal) && external.clientExitedAtMs !== null && external.clientExitedAtMs < Date.parse(terminal.emittedAt), "Independent PID/start observation confirms native client retirement before turn completion");
       check("marker-before-terminal", Boolean(terminal) && external.markerWrittenAtMs !== null && external.markerWrittenAtMs < Date.parse(terminal.emittedAt), "The independent fixture wrote its undisclosed marker before turn completion");
+      const afterCleanupMarkerMatches = await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker);
       const grade = gradeCopilotAttachedSettlement({ expected: copilotOrigin(call!), nativeCall: call ? { ...call, operation: call.operation!, mode: call.mode!, detach: call.detach!, commandSha256: call.commandSha256! } : null,
         expectedCommandSha256: command!.commandSha256, commandExit: external.commandExit,
         expectedShellId: started?.shellId ?? "", nativeShellResult: result ? { ...result, shellId: result.shellId!, commandToolCallId: result.commandToolCallId!, status: result.status!, exitCode: result.exitCode! } : null,
         terminal: terminal ? { observedAtMs: Date.parse(terminal.emittedAt), runId: terminal.runId, turnId: terminal.turnId, status: "succeeded" } : null,
-        cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches: markerMatches });
-      await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches });
+        cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches });
+      await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches, afterCleanupMarkerMatches });
       check("attached-settlement-before-terminal", grade.passed, grade.failures.join(", ") || "Owned finite process and native shell settled before the actual turn terminal");
     }
     await load(); check("one-native-run", runs.length === 1 && runs[0]!.runtimeMode === "native", "Exactly one native run was accounted");