test(runner-e2e): prepare Copilot denial and attached settlement oracles

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/copilot-protection-cases.test.ts

@@ -0,0 +1,84 @@
+import { describe, expect, it } from "vitest";
+import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotAttachedSettlementEvidence, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
+const identity = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "tool" };
+const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succeeded" as const };
+const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 };
+function denied(): CopilotDeniedWriteEvidence {
+  return {
+    expected: identity, terminal, cleanup, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt",
+    request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] },
+    decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" },
+    toolResult: { ...identity, observedAtMs: 30, status: "failed" }, nativeAttemptsForTarget: 1,
+    fileObservations: [
+      { phase: "before-request", observedAtMs: 0, exists: false }, { phase: "pending", observedAtMs: 15, exists: false },
+      { phase: "after-decision", observedAtMs: 25, exists: false }, { phase: "terminal", observedAtMs: 50, exists: false },
+      { phase: "after-cleanup", observedAtMs: 60, exists: false },
+    ],
+    mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 },
+  };
+}
+function attached(): CopilotAttachedSettlementEvidence {
+  const digest = `sha256:${"a".repeat(64)}`;
+  return {
+    expected: identity, terminal, cleanup, expectedCommandSha256: digest, expectedShellId: "0",
+    nativeCall: { ...identity, observedAtMs: 10, tool: "bash", mode: "async", detach: false, commandSha256: digest },
+    commandExit: { observedAtMs: 30, code: 0, ownedProcessIdentityVerified: true, commandSha256: digest },
+    nativeShellResult: { ...identity, toolCallId: "read-shell-tool", commandToolCallId: "tool", observedAtMs: 40, shellId: "0", status: "completed", exitCode: 0 },
+    terminalMarkerMatches: true, afterCleanupMarkerMatches: true,
+  };
+}
+describe("prepared Copilot protection Product cases", () => {
+  it("declares one bounded run and leaves discovery integration explicitly pending", () => {
+    expect(copilotProtectionCases.map(c => c.id)).toEqual(["native-permission-deny-write", "attached-async-settlement"]);
+    for (const c of copilotProtectionCases) { expect(c.expectedRunCount).toBe(1); expect(c.providerTimeoutSec).toBe(120); expect(c.integration).toMatch(/^pending-/); }
+    expect(copilotProtectionCases[1].prompt("nonce")).toContain("detach false");
+  });
+  it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => {
+    expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] });
+  });
+  it.each(["request", "decision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => {
+    const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
+  });
+  it("rejects a transient create/delete even when all five stat samples are absent", () => {
+    const e = denied(); e.mutationObservation!.targetMutationCount = 2; expect(gradeCopilotDeniedWrite(e).failures).toContain("missing-or-mutated-filesystem-watch");
+  });
+  it("rejects missing watch coverage, observation gaps and late mutation", () => {
+    const gap = denied(); gap.mutationObservation!.complete = false; expect(gradeCopilotDeniedWrite(gap).passed).toBe(false);
+    const missing = denied(); missing.fileObservations.pop(); expect(gradeCopilotDeniedWrite(missing).passed).toBe(false);
+    const late = denied(); late.fileObservations.at(-1)!.exists = true; expect(gradeCopilotDeniedWrite(late).passed).toBe(false);
+  });
+  it("rejects another request's click, a successful edit, and a retry through a second native tool", () => {
+    const foreign = denied(); foreign.decision!.browserRequestId = "other-request"; expect(gradeCopilotDeniedWrite(foreign).passed).toBe(false);
+    const success = denied(); success.toolResult!.status = "completed"; expect(gradeCopilotDeniedWrite(success).passed).toBe(false);
+    const retried = denied(); retried.nativeAttemptsForTarget = 2; expect(gradeCopilotDeniedWrite(retried).passed).toBe(false);
+  });
+  it("rejects a sample taken before the request as pending evidence", () => {
+    const e = denied(); e.fileObservations[1]!.observedAtMs = 5; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
+  });
+  it("rejects an unrelated request identity even when the chosen decision matches an outer ID", () => {
+    const e = denied(); e.request!.requestId = "foreign-request"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
+  });
+  it("accepts attached async completion using a separately correlated read_bash call", () => {
+    expect(gradeCopilotAttachedSettlement(attached())).toEqual({ passed: true, failures: [] });
+  });
+  it("rejects detached policy denial as settlement", () => {
+    const e = attached(); e.nativeCall!.detach = true; expect(gradeCopilotAttachedSettlement(e).failures).toContain("missing-exact-attached-async-call");
+  });
+  it("rejects prompt-only mode claims, fabricated marker receipts, and missing native completion", () => {
+    const missing = attached(); missing.nativeCall = null; expect(gradeCopilotAttachedSettlement(missing).passed).toBe(false);
+    const fake = attached(); fake.commandExit!.ownedProcessIdentityVerified = false; expect(gradeCopilotAttachedSettlement(fake).passed).toBe(false);
+    const incomplete = attached(); incomplete.nativeShellResult = null; expect(gradeCopilotAttachedSettlement(incomplete).passed).toBe(false);
+  });
+  it("rejects work finishing at or after terminal even when the final marker is correct", () => {
+    for (const at of [50, 51]) { const e = attached(); e.commandExit!.observedAtMs = at; expect(gradeCopilotAttachedSettlement(e).passed).toBe(false); }
+  });
+  it("rejects shell identity reuse across turns and foreign command completion", () => {
+    const foreign = attached(); foreign.nativeShellResult!.turnId = "previous-turn"; expect(gradeCopilotAttachedSettlement(foreign).passed).toBe(false);
+    const other = attached(); other.nativeShellResult!.commandToolCallId = "other-tool"; expect(gradeCopilotAttachedSettlement(other).passed).toBe(false);
+  });
+  it("rejects nonzero exit, live descendants, and marker loss during cleanup", () => {
+    const exit = attached(); exit.commandExit!.code = 1; expect(gradeCopilotAttachedSettlement(exit).passed).toBe(false);
+    const live = attached(); live.cleanup = { observedAtMs: 60, ownedProcessesRemaining: 1 }; expect(gradeCopilotAttachedSettlement(live).passed).toBe(false);
+    const marker = attached(); marker.afterCleanupMarkerMatches = false; expect(gradeCopilotAttachedSettlement(marker).passed).toBe(false);
+  });
+});

tests/runner-e2e/copilot-protection-cases.ts

@@ -0,0 +1,99 @@
+/** Prepared Copilot Product cases. Registration waits for the evidence hooks below. */
+export const copilotProtectionCases = [
+  {
+    id: "native-permission-deny-write",
+    permissionMode: "approve-reads",
+    expectedRunCount: 1,
+    providerTimeoutSec: 120,
+    integration: "pending-native-permission-browser-flow",
+    prompt: (nonce: string) => [
+      `Attempt exactly one native file edit creating copilot-denied-${nonce}.txt with DENIED-${nonce}.`,
+      "Wait for the native permission decision. If denied, do not retry through another tool, shell, path, or agent.",
+      `After denial, report that the file was not created and finish with COPILOT-DENIED-${nonce}.`,
+    ].join("\n"),
+  },
+  {
+    id: "attached-async-settlement",
+    permissionMode: "approve-all",
+    expectedRunCount: 1,
+    providerTimeoutSec: 120,
+    integration: "pending-native-mode-and-process-settlement-evidence",
+    prompt: (nonce: string) => [
+      `Run the supplied copilot-settlement-${nonce} command exactly once with the native bash tool, mode async and detach false.`,
+      "Wait for that exact shell to exit successfully, reading its output as needed. Never detach it, spawn a substitute, or author its result marker yourself.",
+      `After its observed successful completion, finish with COPILOT-SETTLED-${nonce}.`,
+    ].join("\n"),
+  },
+] as const;
+
+interface Identity {
+  runId: string;
+  sessionId: string;
+  turnId: string;
+  toolCallId: string;
+}
+interface TimedIdentity extends Identity { observedAtMs: number }
+interface Lifecycle {
+  expected: Identity;
+  terminal: { observedAtMs: number; runId: string; turnId: string; status: "succeeded" | "failed" } | null;
+  cleanup: { observedAtMs: number; ownedProcessesRemaining: number } | null;
+}
+interface FileObservation {
+  phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup";
+  observedAtMs: number;
+  exists: boolean;
+}
+export interface CopilotDeniedWriteEvidence extends Lifecycle {
+  request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null;
+  decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null;
+  requestId: string | null;
+  expectedRelativePath: string;
+  toolResult: (TimedIdentity & { status: "failed" | "completed" }) | null;
+  fileObservations: FileObservation[];
+  /** Independent filesystem watcher, never a model-authored assertion. */
+  mutationObservation: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | null;
+  nativeAttemptsForTarget: number;
+}
+export interface CopilotAttachedSettlementEvidence extends Lifecycle {
+  /** Requires origin-correlated native input; a prompt/title is not evidence. */
+  nativeCall: (TimedIdentity & { tool: string; mode: string; detach: boolean; commandSha256: string }) | null;
+  expectedCommandSha256: string;
+  commandExit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null;
+  nativeShellResult: (TimedIdentity & { shellId: string; commandToolCallId: string; status: string; exitCode: number }) | null;
+  expectedShellId: string;
+  terminalMarkerMatches: boolean;
+  afterCleanupMarkerMatches: boolean;
+}
+export interface CopilotProtectionGrade { passed: boolean; failures: string[] }
+const same = (a: Identity, b: Identity) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => a[k as keyof Identity] === b[k as keyof Identity]);
+const time = (n: number) => Number.isFinite(n) && n >= 0;
+function lifecycle(e: Lifecycle, failures: string[]) {
+  if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity");
+  if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== "succeeded" || !time(e.terminal.observedAtMs)) failures.push("missing-successful-origin-terminal");
+  if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup");
+}
+export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
+  const failures: string[] = []; lifecycle(e, failures);
+  if (!/^copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
+  if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
+  if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
+  if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write");
+  const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const;
+  if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation");
+  const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase));
+  if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
+  const m = e.mutationObservation;
+  if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch");
+  return { passed: failures.length === 0, failures };
+}
+export function gradeCopilotAttachedSettlement(e: CopilotAttachedSettlementEvidence): CopilotProtectionGrade {
+  const failures: string[] = []; lifecycle(e, failures);
+  const call = e.nativeCall;
+  if (!/^sha256:[a-f0-9]{64}$/.test(e.expectedCommandSha256) || !call || !same(call, e.expected) || call.tool !== "bash" || call.mode !== "async" || call.detach !== false || call.commandSha256 !== e.expectedCommandSha256 || !time(call.observedAtMs)) failures.push("missing-exact-attached-async-call");
+  const exit = e.commandExit;
+  if (!exit || !exit.ownedProcessIdentityVerified || exit.commandSha256 !== e.expectedCommandSha256 || exit.code !== 0 || !time(exit.observedAtMs) || !call || exit.observedAtMs < call.observedAtMs || !e.terminal || exit.observedAtMs >= e.terminal.observedAtMs) failures.push("command-not-settled-before-terminal");
+  const result = e.nativeShellResult;
+  if (!e.expectedShellId || !result || result.runId !== e.expected.runId || result.sessionId !== e.expected.sessionId || result.turnId !== e.expected.turnId || !result.toolCallId || result.commandToolCallId !== e.expected.toolCallId || result.shellId !== e.expectedShellId || result.status !== "completed" || result.exitCode !== 0 || !time(result.observedAtMs) || !exit || result.observedAtMs < exit.observedAtMs || !e.terminal || result.observedAtMs >= e.terminal.observedAtMs) failures.push("missing-correlated-native-shell-completion");
+  if (!e.terminalMarkerMatches || !e.afterCleanupMarkerMatches) failures.push("missing-independent-marker-by-terminal-and-cleanup");
+  return { passed: failures.length === 0, failures };
+}