fix(evals): correlate denied Copilot tool target by origin

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/copilot-protection-evidence.test.ts

@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
 import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
-import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
+import { countCopilotEditOriginsForTarget, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
 import type { CopilotToolNotice } from "./copilot-evidence.js";
 const notice: CopilotToolNotice = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "edit", stage: "tool", operation: "edit", observedAtMs: 1, seq: 1 };
 describe("Copilot protection independent evidence", () => {
@@ -26,6 +26,54 @@ describe("Copilot protection independent evidence", () => {
 
 import { assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotActionNotices, copilotRemoteDeniedSample, prepareCopilotRemoteAction, type CopilotRemoteSnapshot, type CopilotRemoteFixture } from "./copilot-protection-evidence.js";
 const target = "copilot-denied-nonce.txt";
+describe("Copilot denied-target origin correlation", () => {
+  // Captured native shape: updates omit the target; permission events bind it.
+  const deniedTarget = "pc-denied-fixture/copilot-denied-nonce.txt";
+  const captured = (): CopilotToolNotice[] => [
+    { ...notice, seq: 39, status: "pending" },
+    { ...notice, seq: 42, stage: "permission_requested", requestId: "request", target: deniedTarget, declineOffered: true },
+    { ...notice, seq: 44, stage: "permission_delivered", requestId: "request", target: deniedTarget, outcome: "reject_once" },
+    { ...notice, seq: 45, status: "failed" },
+  ];
+  it("counts the captured denied edit across exact permission and tool origins", () => {
+    expect(countCopilotEditOriginsForTarget(captured(), deniedTarget)).toBe(1);
+    expect(countCopilotToolOrigins(copilotActionNotices(captured(), captured()[0]!))).toBe(1);
+  });
+  it.each(["runId", "sessionId", "turnId", "toolCallId"] as const)("cannot borrow a target across %s", field => {
+    const rows = captured().map(n => n.stage === "tool" ? { ...n, [field]: "foreign" } : n);
+    expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(0);
+  });
+  it.each(["in_progress", "completed", "failed"] as const)("counts a retry first seen as %s", status => {
+    const rows = [...captured(), { ...notice, toolCallId: "retry", seq: 46, target: deniedTarget, status }];
+    expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(2);
+    expect(countCopilotToolOrigins(copilotActionNotices(rows, rows[0]!))).toBe(2);
+  });
+  it("does not hide an alternate operation with no target", () => {
+    const rows = [...captured(), { ...notice, toolCallId: "alternate", seq: 46, operation: "execute" as const }];
+    expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(1);
+    expect(countCopilotToolOrigins(copilotActionNotices(rows, rows[0]!))).toBe(2);
+  });
+  it("does not count permission-only or entirely unbound evidence", () => {
+    expect(countCopilotEditOriginsForTarget(captured().filter(n => n.stage !== "tool"), deniedTarget)).toBe(0);
+    expect(countCopilotEditOriginsForTarget(captured().filter(n => n.stage === "tool"), deniedTarget)).toBe(0);
+  });
+  it.each([
+    { target: "other.txt" }, { operation: "execute" as const },
+  ])("rejects conflicting tool evidence %j", conflict => {
+    const rows = captured(); rows[3] = { ...rows[3]!, ...conflict };
+    expect(() => countCopilotEditOriginsForTarget(rows, deniedTarget)).toThrow(/Conflicting/);
+  });
+  it("rejects another permission request on the same tool origin", () => {
+    const rows = captured(); rows[2] = { ...rows[2]!, requestId: "other" };
+    expect(() => countCopilotEditOriginsForTarget(rows, deniedTarget)).toThrow(/permission/);
+    expect(() => countCopilotEditOriginsForTarget([...captured(), { ...captured()[1]!, seq: 46 }], deniedTarget)).toThrow(/permission/);
+  });
+  it("rejects a restarted or repeated native lifecycle under a reused origin", () => {
+    for (const status of ["pending", "in_progress", "completed", "failed"] as const) {
+      expect(() => countCopilotEditOriginsForTarget([...captured(), { ...notice, status, seq: 46 }], deniedTarget)).toThrow(/lifecycle/);
+    }
+  });
+});
 function receipt(): CopilotRemoteSnapshot {
   const root = { pid: 50, ppid: 1, startTicks: "200", bootId: "12345678-1234-1234-1234-123456789abc" };
   return { binding: { companyId: "company", environmentId: "env", runId: "run", leaseId: "lease", sandboxId: "sandbox", image: `image@sha256:${"a".repeat(64)}`, remoteCwd: "/home/daytona/workspace" },

tests/runner-e2e/copilot-protection-evidence.ts

@@ -8,6 +8,39 @@ export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]):
   return new Set(notices.filter(n => n.stage === "tool").map(n => JSON.stringify([n.runId, n.sessionId, n.turnId, n.toolCallId]))).size;
 }
 
+/** Permission requests may carry the target omitted by native tool updates. */
+export function countCopilotEditOriginsForTarget(notices: readonly CopilotToolNotice[], target: string): number {
+  const origins = new Map<string, CopilotToolNotice[]>();
+  for (const notice of notices) {
+    const key = JSON.stringify([notice.runId, notice.sessionId, notice.turnId, notice.toolCallId]);
+    const group = origins.get(key) ?? [];
+    group.push(notice); origins.set(key, group);
+  }
+  let count = 0;
+  for (const group of origins.values()) {
+    if (!group.some(n => n.target === target)) continue;
+    if (group.some(n => (n.target !== undefined && n.target !== target) || (n.operation !== undefined && n.operation !== "edit"))) {
+      throw new Error("Conflicting Copilot denied-target origin evidence");
+    }
+    const permissions = group.filter(n => n.stage !== "tool");
+    const requestIds = new Set(permissions.map(n => n.requestId));
+    if (requestIds.has(undefined) || requestIds.size > 1
+      || permissions.filter(n => n.stage === "permission_requested").length > 1
+      || permissions.filter(n => n.stage === "permission_delivered").length > 1) {
+      throw new Error("Repeated or unbound Copilot denied-target permission");
+    }
+    const tools = group.filter(n => n.stage === "tool");
+    const terminal = tools.filter(n => n.status === "completed" || n.status === "failed");
+    if (terminal.length > 1 || tools.filter(n => n.status === "pending").length > 1
+      || (terminal[0] && tools.some(n => n.seq > terminal[0]!.seq))) {
+      throw new Error("Repeated Copilot denied-target tool lifecycle");
+    }
+    // A permission notice alone never proves that a native tool was attempted.
+    if (tools.some(n => n.operation === "edit")) count++;
+  }
+  return count;
+}
+
 /** Re-read independently after cleanup; a pre-cleanup value is not evidence. */
 export async function readCopilotMarkerAfterCleanup(close: () => Promise<void>, path: string, expected: string): Promise<boolean> {
   await close();

tests/runner-e2e/copilot-protection-flow.ts

@@ -8,7 +8,7 @@ import { createTaskThroughUi } from "./user-actions.js";
 import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
 import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js";
 import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
-import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
+import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
 import type { LiveFixtureValues } from "./live-fixtures.js";
 import type { MatrixExecution } from "./types.js";
 type Row = Record<string, any>;
@@ -149,7 +149,7 @@ export async function runCopilotProtectionFlow(input: {
         terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status },
         cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope },
         cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt,
-        nativeAttemptsForTarget: countCopilotToolOrigins(notices.filter(n => n.operation === "edit" && n.target === target)),
+        nativeAttemptsForTarget: countCopilotEditOriginsForTarget(notices, target),
       };
       await input.evidence("copilot-denial-proof.json", { evidence, processes, notices });
       const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup");