diff --git a/tests/runner-e2e/copilot-protection-cases.test.ts b/tests/runner-e2e/copilot-protection-cases.test.ts new file mode 100644 index 0000000000..aff95d2938 --- /dev/null +++ b/tests/runner-e2e/copilot-protection-cases.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotAttachedSettlementEvidence, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; +const identity = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "tool" }; +const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succeeded" as const }; +const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 }; +function denied(): CopilotDeniedWriteEvidence { + return { + expected: identity, terminal, cleanup, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", + request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] }, + decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" }, + toolResult: { ...identity, observedAtMs: 30, status: "failed" }, nativeAttemptsForTarget: 1, + fileObservations: [ + { phase: "before-request", observedAtMs: 0, exists: false }, { phase: "pending", observedAtMs: 15, exists: false }, + { phase: "after-decision", observedAtMs: 25, exists: false }, { phase: "terminal", observedAtMs: 50, exists: false }, + { phase: "after-cleanup", observedAtMs: 60, exists: false }, + ], + mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 }, + }; +} +function attached(): CopilotAttachedSettlementEvidence { + const digest = `sha256:${"a".repeat(64)}`; + return { + expected: identity, terminal, cleanup, expectedCommandSha256: digest, expectedShellId: "0", + nativeCall: { ...identity, observedAtMs: 10, tool: "bash", mode: "async", detach: false, commandSha256: digest }, + commandExit: { observedAtMs: 30, code: 0, ownedProcessIdentityVerified: true, commandSha256: digest }, + nativeShellResult: { ...identity, toolCallId: "read-shell-tool", commandToolCallId: "tool", observedAtMs: 40, shellId: "0", status: "completed", exitCode: 0 }, + terminalMarkerMatches: true, afterCleanupMarkerMatches: true, + }; +} +describe("prepared Copilot protection Product cases", () => { + it("declares one bounded run and leaves discovery integration explicitly pending", () => { + expect(copilotProtectionCases.map(c => c.id)).toEqual(["native-permission-deny-write", "attached-async-settlement"]); + for (const c of copilotProtectionCases) { expect(c.expectedRunCount).toBe(1); expect(c.providerTimeoutSec).toBe(120); expect(c.integration).toMatch(/^pending-/); } + expect(copilotProtectionCases[1].prompt("nonce")).toContain("detach false"); + }); + it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => { + expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] }); + }); + it.each(["request", "decision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => { + const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + }); + it("rejects a transient create/delete even when all five stat samples are absent", () => { + const e = denied(); e.mutationObservation!.targetMutationCount = 2; expect(gradeCopilotDeniedWrite(e).failures).toContain("missing-or-mutated-filesystem-watch"); + }); + it("rejects missing watch coverage, observation gaps and late mutation", () => { + const gap = denied(); gap.mutationObservation!.complete = false; expect(gradeCopilotDeniedWrite(gap).passed).toBe(false); + const missing = denied(); missing.fileObservations.pop(); expect(gradeCopilotDeniedWrite(missing).passed).toBe(false); + const late = denied(); late.fileObservations.at(-1)!.exists = true; expect(gradeCopilotDeniedWrite(late).passed).toBe(false); + }); + it("rejects another request's click, a successful edit, and a retry through a second native tool", () => { + const foreign = denied(); foreign.decision!.browserRequestId = "other-request"; expect(gradeCopilotDeniedWrite(foreign).passed).toBe(false); + const success = denied(); success.toolResult!.status = "completed"; expect(gradeCopilotDeniedWrite(success).passed).toBe(false); + const retried = denied(); retried.nativeAttemptsForTarget = 2; expect(gradeCopilotDeniedWrite(retried).passed).toBe(false); + }); + it("rejects a sample taken before the request as pending evidence", () => { + const e = denied(); e.fileObservations[1]!.observedAtMs = 5; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid"); + }); + it("rejects an unrelated request identity even when the chosen decision matches an outer ID", () => { + const e = denied(); e.request!.requestId = "foreign-request"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + }); + it("accepts attached async completion using a separately correlated read_bash call", () => { + expect(gradeCopilotAttachedSettlement(attached())).toEqual({ passed: true, failures: [] }); + }); + it("rejects detached policy denial as settlement", () => { + const e = attached(); e.nativeCall!.detach = true; expect(gradeCopilotAttachedSettlement(e).failures).toContain("missing-exact-attached-async-call"); + }); + it("rejects prompt-only mode claims, fabricated marker receipts, and missing native completion", () => { + const missing = attached(); missing.nativeCall = null; expect(gradeCopilotAttachedSettlement(missing).passed).toBe(false); + const fake = attached(); fake.commandExit!.ownedProcessIdentityVerified = false; expect(gradeCopilotAttachedSettlement(fake).passed).toBe(false); + const incomplete = attached(); incomplete.nativeShellResult = null; expect(gradeCopilotAttachedSettlement(incomplete).passed).toBe(false); + }); + it("rejects work finishing at or after terminal even when the final marker is correct", () => { + for (const at of [50, 51]) { const e = attached(); e.commandExit!.observedAtMs = at; expect(gradeCopilotAttachedSettlement(e).passed).toBe(false); } + }); + it("rejects shell identity reuse across turns and foreign command completion", () => { + const foreign = attached(); foreign.nativeShellResult!.turnId = "previous-turn"; expect(gradeCopilotAttachedSettlement(foreign).passed).toBe(false); + const other = attached(); other.nativeShellResult!.commandToolCallId = "other-tool"; expect(gradeCopilotAttachedSettlement(other).passed).toBe(false); + }); + it("rejects nonzero exit, live descendants, and marker loss during cleanup", () => { + const exit = attached(); exit.commandExit!.code = 1; expect(gradeCopilotAttachedSettlement(exit).passed).toBe(false); + const live = attached(); live.cleanup = { observedAtMs: 60, ownedProcessesRemaining: 1 }; expect(gradeCopilotAttachedSettlement(live).passed).toBe(false); + const marker = attached(); marker.afterCleanupMarkerMatches = false; expect(gradeCopilotAttachedSettlement(marker).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/copilot-protection-cases.ts b/tests/runner-e2e/copilot-protection-cases.ts new file mode 100644 index 0000000000..6915da22fb --- /dev/null +++ b/tests/runner-e2e/copilot-protection-cases.ts @@ -0,0 +1,99 @@ +/** Prepared Copilot Product cases. Registration waits for the evidence hooks below. */ +export const copilotProtectionCases = [ + { + id: "native-permission-deny-write", + permissionMode: "approve-reads", + expectedRunCount: 1, + providerTimeoutSec: 120, + integration: "pending-native-permission-browser-flow", + prompt: (nonce: string) => [ + `Attempt exactly one native file edit creating copilot-denied-${nonce}.txt with DENIED-${nonce}.`, + "Wait for the native permission decision. If denied, do not retry through another tool, shell, path, or agent.", + `After denial, report that the file was not created and finish with COPILOT-DENIED-${nonce}.`, + ].join("\n"), + }, + { + id: "attached-async-settlement", + permissionMode: "approve-all", + expectedRunCount: 1, + providerTimeoutSec: 120, + integration: "pending-native-mode-and-process-settlement-evidence", + prompt: (nonce: string) => [ + `Run the supplied copilot-settlement-${nonce} command exactly once with the native bash tool, mode async and detach false.`, + "Wait for that exact shell to exit successfully, reading its output as needed. Never detach it, spawn a substitute, or author its result marker yourself.", + `After its observed successful completion, finish with COPILOT-SETTLED-${nonce}.`, + ].join("\n"), + }, +] as const; + +interface Identity { + runId: string; + sessionId: string; + turnId: string; + toolCallId: string; +} +interface TimedIdentity extends Identity { observedAtMs: number } +interface Lifecycle { + expected: Identity; + terminal: { observedAtMs: number; runId: string; turnId: string; status: "succeeded" | "failed" } | null; + cleanup: { observedAtMs: number; ownedProcessesRemaining: number } | null; +} +interface FileObservation { + phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup"; + observedAtMs: number; + exists: boolean; +} +export interface CopilotDeniedWriteEvidence extends Lifecycle { + request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null; + decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null; + requestId: string | null; + expectedRelativePath: string; + toolResult: (TimedIdentity & { status: "failed" | "completed" }) | null; + fileObservations: FileObservation[]; + /** Independent filesystem watcher, never a model-authored assertion. */ + mutationObservation: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | null; + nativeAttemptsForTarget: number; +} +export interface CopilotAttachedSettlementEvidence extends Lifecycle { + /** Requires origin-correlated native input; a prompt/title is not evidence. */ + nativeCall: (TimedIdentity & { tool: string; mode: string; detach: boolean; commandSha256: string }) | null; + expectedCommandSha256: string; + commandExit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null; + nativeShellResult: (TimedIdentity & { shellId: string; commandToolCallId: string; status: string; exitCode: number }) | null; + expectedShellId: string; + terminalMarkerMatches: boolean; + afterCleanupMarkerMatches: boolean; +} +export interface CopilotProtectionGrade { passed: boolean; failures: string[] } +const same = (a: Identity, b: Identity) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => a[k as keyof Identity] === b[k as keyof Identity]); +const time = (n: number) => Number.isFinite(n) && n >= 0; +function lifecycle(e: Lifecycle, failures: string[]) { + if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity"); + if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== "succeeded" || !time(e.terminal.observedAtMs)) failures.push("missing-successful-origin-terminal"); + if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup"); +} +export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade { + const failures: string[] = []; lifecycle(e, failures); + if (!/^copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request"); + if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial"); + if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal"); + if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write"); + const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const; + if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation"); + const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase)); + if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid"); + const m = e.mutationObservation; + if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch"); + return { passed: failures.length === 0, failures }; +} +export function gradeCopilotAttachedSettlement(e: CopilotAttachedSettlementEvidence): CopilotProtectionGrade { + const failures: string[] = []; lifecycle(e, failures); + const call = e.nativeCall; + if (!/^sha256:[a-f0-9]{64}$/.test(e.expectedCommandSha256) || !call || !same(call, e.expected) || call.tool !== "bash" || call.mode !== "async" || call.detach !== false || call.commandSha256 !== e.expectedCommandSha256 || !time(call.observedAtMs)) failures.push("missing-exact-attached-async-call"); + const exit = e.commandExit; + if (!exit || !exit.ownedProcessIdentityVerified || exit.commandSha256 !== e.expectedCommandSha256 || exit.code !== 0 || !time(exit.observedAtMs) || !call || exit.observedAtMs < call.observedAtMs || !e.terminal || exit.observedAtMs >= e.terminal.observedAtMs) failures.push("command-not-settled-before-terminal"); + const result = e.nativeShellResult; + if (!e.expectedShellId || !result || result.runId !== e.expected.runId || result.sessionId !== e.expected.sessionId || result.turnId !== e.expected.turnId || !result.toolCallId || result.commandToolCallId !== e.expected.toolCallId || result.shellId !== e.expectedShellId || result.status !== "completed" || result.exitCode !== 0 || !time(result.observedAtMs) || !exit || result.observedAtMs < exit.observedAtMs || !e.terminal || result.observedAtMs >= e.terminal.observedAtMs) failures.push("missing-correlated-native-shell-completion"); + if (!e.terminalMarkerMatches || !e.afterCleanupMarkerMatches) failures.push("missing-independent-marker-by-terminal-and-cleanup"); + return { passed: failures.length === 0, failures }; +}