diff --git a/tests/runner-e2e/copilot-protection-evidence.test.ts b/tests/runner-e2e/copilot-protection-evidence.test.ts index e58e54e261f..751aafb9076 100644 --- a/tests/runner-e2e/copilot-protection-evidence.test.ts +++ b/tests/runner-e2e/copilot-protection-evidence.test.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; +import { countCopilotEditOriginsForTarget, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { CopilotToolNotice } from "./copilot-evidence.js"; const notice: CopilotToolNotice = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "edit", stage: "tool", operation: "edit", observedAtMs: 1, seq: 1 }; describe("Copilot protection independent evidence", () => { @@ -26,6 +26,54 @@ describe("Copilot protection independent evidence", () => { import { assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotActionNotices, copilotRemoteDeniedSample, prepareCopilotRemoteAction, type CopilotRemoteSnapshot, type CopilotRemoteFixture } from "./copilot-protection-evidence.js"; const target = "copilot-denied-nonce.txt"; +describe("Copilot denied-target origin correlation", () => { + // Captured native shape: updates omit the target; permission events bind it. + const deniedTarget = "pc-denied-fixture/copilot-denied-nonce.txt"; + const captured = (): CopilotToolNotice[] => [ + { ...notice, seq: 39, status: "pending" }, + { ...notice, seq: 42, stage: "permission_requested", requestId: "request", target: deniedTarget, declineOffered: true }, + { ...notice, seq: 44, stage: "permission_delivered", requestId: "request", target: deniedTarget, outcome: "reject_once" }, + { ...notice, seq: 45, status: "failed" }, + ]; + it("counts the captured denied edit across exact permission and tool origins", () => { + expect(countCopilotEditOriginsForTarget(captured(), deniedTarget)).toBe(1); + expect(countCopilotToolOrigins(copilotActionNotices(captured(), captured()[0]!))).toBe(1); + }); + it.each(["runId", "sessionId", "turnId", "toolCallId"] as const)("cannot borrow a target across %s", field => { + const rows = captured().map(n => n.stage === "tool" ? { ...n, [field]: "foreign" } : n); + expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(0); + }); + it.each(["in_progress", "completed", "failed"] as const)("counts a retry first seen as %s", status => { + const rows = [...captured(), { ...notice, toolCallId: "retry", seq: 46, target: deniedTarget, status }]; + expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(2); + expect(countCopilotToolOrigins(copilotActionNotices(rows, rows[0]!))).toBe(2); + }); + it("does not hide an alternate operation with no target", () => { + const rows = [...captured(), { ...notice, toolCallId: "alternate", seq: 46, operation: "execute" as const }]; + expect(countCopilotEditOriginsForTarget(rows, deniedTarget)).toBe(1); + expect(countCopilotToolOrigins(copilotActionNotices(rows, rows[0]!))).toBe(2); + }); + it("does not count permission-only or entirely unbound evidence", () => { + expect(countCopilotEditOriginsForTarget(captured().filter(n => n.stage !== "tool"), deniedTarget)).toBe(0); + expect(countCopilotEditOriginsForTarget(captured().filter(n => n.stage === "tool"), deniedTarget)).toBe(0); + }); + it.each([ + { target: "other.txt" }, { operation: "execute" as const }, + ])("rejects conflicting tool evidence %j", conflict => { + const rows = captured(); rows[3] = { ...rows[3]!, ...conflict }; + expect(() => countCopilotEditOriginsForTarget(rows, deniedTarget)).toThrow(/Conflicting/); + }); + it("rejects another permission request on the same tool origin", () => { + const rows = captured(); rows[2] = { ...rows[2]!, requestId: "other" }; + expect(() => countCopilotEditOriginsForTarget(rows, deniedTarget)).toThrow(/permission/); + expect(() => countCopilotEditOriginsForTarget([...captured(), { ...captured()[1]!, seq: 46 }], deniedTarget)).toThrow(/permission/); + }); + it("rejects a restarted or repeated native lifecycle under a reused origin", () => { + for (const status of ["pending", "in_progress", "completed", "failed"] as const) { + expect(() => countCopilotEditOriginsForTarget([...captured(), { ...notice, status, seq: 46 }], deniedTarget)).toThrow(/lifecycle/); + } + }); +}); function receipt(): CopilotRemoteSnapshot { const root = { pid: 50, ppid: 1, startTicks: "200", bootId: "12345678-1234-1234-1234-123456789abc" }; return { binding: { companyId: "company", environmentId: "env", runId: "run", leaseId: "lease", sandboxId: "sandbox", image: `image@sha256:${"a".repeat(64)}`, remoteCwd: "/home/daytona/workspace" }, diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 61ce447f7e8..68899445ede 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -8,6 +8,39 @@ export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): return new Set(notices.filter(n => n.stage === "tool").map(n => JSON.stringify([n.runId, n.sessionId, n.turnId, n.toolCallId]))).size; } +/** Permission requests may carry the target omitted by native tool updates. */ +export function countCopilotEditOriginsForTarget(notices: readonly CopilotToolNotice[], target: string): number { + const origins = new Map(); + for (const notice of notices) { + const key = JSON.stringify([notice.runId, notice.sessionId, notice.turnId, notice.toolCallId]); + const group = origins.get(key) ?? []; + group.push(notice); origins.set(key, group); + } + let count = 0; + for (const group of origins.values()) { + if (!group.some(n => n.target === target)) continue; + if (group.some(n => (n.target !== undefined && n.target !== target) || (n.operation !== undefined && n.operation !== "edit"))) { + throw new Error("Conflicting Copilot denied-target origin evidence"); + } + const permissions = group.filter(n => n.stage !== "tool"); + const requestIds = new Set(permissions.map(n => n.requestId)); + if (requestIds.has(undefined) || requestIds.size > 1 + || permissions.filter(n => n.stage === "permission_requested").length > 1 + || permissions.filter(n => n.stage === "permission_delivered").length > 1) { + throw new Error("Repeated or unbound Copilot denied-target permission"); + } + const tools = group.filter(n => n.stage === "tool"); + const terminal = tools.filter(n => n.status === "completed" || n.status === "failed"); + if (terminal.length > 1 || tools.filter(n => n.status === "pending").length > 1 + || (terminal[0] && tools.some(n => n.seq > terminal[0]!.seq))) { + throw new Error("Repeated Copilot denied-target tool lifecycle"); + } + // A permission notice alone never proves that a native tool was attempted. + if (tools.some(n => n.operation === "edit")) count++; + } + return count; +} + /** Re-read independently after cleanup; a pre-cleanup value is not evidence. */ export async function readCopilotMarkerAfterCleanup(close: () => Promise, path: string, expected: string): Promise { await close(); diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 2c1dde732e5..a3808aaeacf 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -8,7 +8,7 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; -import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; +import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; @@ -149,7 +149,7 @@ export async function runCopilotProtectionFlow(input: { terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status }, cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope }, cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt, - nativeAttemptsForTarget: countCopilotToolOrigins(notices.filter(n => n.operation === "edit" && n.target === target)), + nativeAttemptsForTarget: countCopilotEditOriginsForTarget(notices, target), }; await input.evidence("copilot-denial-proof.json", { evidence, processes, notices }); const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup");