diff --git a/tests/runner-e2e/copilot-local-fixtures.ts b/tests/runner-e2e/copilot-local-fixtures.ts index d54be65409..0179869c22 100644 --- a/tests/runner-e2e/copilot-local-fixtures.ts +++ b/tests/runner-e2e/copilot-local-fixtures.ts @@ -3,31 +3,71 @@ import { createHash, randomBytes } from "node:crypto"; import { watch, lstatSync, type FSWatcher } from "node:fs"; import { lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type Socket } from "node:net"; -import { join } from "node:path"; +import { basename, join, relative } from "node:path"; export const sha256 = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; export async function exists(path: string): Promise { try { await lstat(path); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } } +/** Allocate before dispatch; ordinary workspace startup cannot mutate this parent. */ +export async function createDeniedTargetFixture(workspacePath: string, name: string) { + if (!name || basename(name) !== name || name === "." || name === "..") throw new Error("Invalid denied target name"); + const directory = await mkdtemp(join(workspacePath, "pc-denied-")); + const targetPath = join(directory, name); + return { directory, targetPath, targetRelativePath: relative(workspacePath, targetPath), watcher: watchDeniedTarget(directory, name) }; +} + +/** Substitute the one authored target, never append a contradictory second path. */ +export function bindDeniedTargetPrompt(prompt: string, original: string, target: string): string { + const parts = prompt.split(original); + if (parts.length !== 2) throw new Error("Denied prompt must name its exact target once"); + return parts.join(target); +} + export function watchDeniedTarget(directory: string, name: string) { - const startedAtMs = Date.now(); let complete = true, targetMutationCount = 0; + if (!name || basename(name) !== name || name === "." || name === "..") throw new Error("Invalid denied target name"); + const startedAtMs = Date.now(); let targetMutationCount = 0; + const reasons = new Set(); const before = lstatSync(directory, { bigint: true }); - let final: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | undefined; - const watcher: FSWatcher = watch(directory, (_kind, filename) => { - if (filename === null) complete = false; - else if (String(filename) === name) targetMutationCount++; + if (!before.isDirectory() || before.isSymbolicLink()) throw new Error("Denied target parent must be a real directory"); + const targetAbsent = () => { try { lstatSync(join(directory, name)); return false; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; throw error; } }; + if (!targetAbsent()) throw new Error("Denied target must initially be absent"); + const identity = (stat: import("node:fs").BigIntStats) => ({ dev: String(stat.dev), ino: String(stat.ino), mtimeNs: String(stat.mtimeNs), ctimeNs: String(stat.ctimeNs) }); + const events: Array<{ sequence: number; observedAtMs: number; kind: string; target: boolean; filenameKnown: boolean }> = []; + let eventCount = 0, lastEventAtMs = startedAtMs, closing = false; + let final: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number; reasons: string[]; initialParent: ReturnType; finalParent: ReturnType | null; events: typeof events } | undefined; + const watcher: FSWatcher = watch(directory, (kind, filename) => { + const observedAtMs = Date.now(); + if (observedAtMs < lastEventAtMs) reasons.add("event-order-invalid"); + lastEventAtMs = observedAtMs; + const target = filename !== null && String(filename) === name; + if (filename === null) reasons.add("event-filename-missing"); + if (target) targetMutationCount++; + if (++eventCount <= 128) events.push({ sequence: eventCount, observedAtMs, kind, target, filenameKnown: filename !== null }); + else reasons.add("event-journal-overflow"); }); - watcher.on("error", () => { complete = false; }); + watcher.on("error", () => { reasons.add("watch-error"); }); + watcher.on("close", () => { if (!closing) reasons.add("watch-closed-before-finish"); }); + // Pin both identity and directory version across watcher installation. + try { + const armed = lstatSync(directory, { bigint: true }); + if (!armed.isDirectory() || armed.dev !== before.dev || armed.ino !== before.ino) reasons.add("parent-identity-changed-during-arm"); + if (armed.mtimeNs !== before.mtimeNs || armed.ctimeNs !== before.ctimeNs || !targetAbsent()) reasons.add("coverage-gap-during-arm"); + } catch { reasons.add("parent-unavailable-during-arm"); } return { finish() { if (final) return final; let after: import("node:fs").BigIntStats | undefined; - try { after = lstatSync(directory, { bigint: true }); } catch { complete = false; } + try { after = lstatSync(directory, { bigint: true }); } catch { reasons.add("parent-unavailable-at-finish"); } // FSEvents may coalesce a rapid create/delete. A changed directory version // with no attributed event is a coverage gap, never proof of no mutation. - if (!after || after.dev !== before.dev || after.ino !== before.ino || !after.isDirectory()) complete = false; - if (after && (after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) && targetMutationCount === 0) complete = false; - final = { startedAtMs, endedAtMs: Date.now(), complete, targetMutationCount }; watcher.close(); return final; + if (after && (after.dev !== before.dev || after.ino !== before.ino || !after.isDirectory())) reasons.add("parent-identity-changed"); + if (after && (after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) && targetMutationCount === 0) reasons.add("coverage-gap-parent-version-changed"); + try { if (!targetAbsent() && targetMutationCount === 0) reasons.add("coverage-gap-unobserved-target"); } catch { reasons.add("target-unavailable-at-finish"); } + const endedAtMs = Date.now(); + if (endedAtMs < lastEventAtMs) reasons.add("event-order-invalid"); + final = { startedAtMs, endedAtMs, complete: reasons.size === 0, targetMutationCount, reasons: [...reasons], initialParent: identity(before), finalParent: after ? identity(after) : null, events }; + closing = true; watcher.close(); return final; } }; } interface ProcessIdentity { pid: number; parent: number; start: string } diff --git a/tests/runner-e2e/copilot-protection-cases.test.ts b/tests/runner-e2e/copilot-protection-cases.test.ts index 6c414f2476..af60cb0906 100644 --- a/tests/runner-e2e/copilot-protection-cases.test.ts +++ b/tests/runner-e2e/copilot-protection-cases.test.ts @@ -37,6 +37,16 @@ describe("Copilot protection Product oracles", () => { it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => { expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] }); }); + it("requires the exact isolated target in native permission evidence", () => { + const e = denied(); e.expectedRelativePath = "pc-denied-ABC123/copilot-denied-nonce.txt"; + expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + e.request!.targetRelativePath = e.expectedRelativePath; + expect(gradeCopilotDeniedWrite(e).passed).toBe(true); + for (const path of ["../copilot-denied-nonce.txt", "other/copilot-denied-nonce.txt", "pc-denied-ABC123/../copilot-denied-nonce.txt"]) { + e.expectedRelativePath = path; e.request!.targetRelativePath = path; + expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + } + }); it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => { const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); }); diff --git a/tests/runner-e2e/copilot-protection-cases.ts b/tests/runner-e2e/copilot-protection-cases.ts index 6dacf6d4ee..f2aca1a826 100644 --- a/tests/runner-e2e/copilot-protection-cases.ts +++ b/tests/runner-e2e/copilot-protection-cases.ts @@ -77,7 +77,7 @@ function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded" export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade { const failures: string[] = []; lifecycle(e, failures, "cancelled"); if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation"); - if (!/^copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request"); + if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request"); if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial"); if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection"); if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal"); diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index d168caefd2..155fb0c17f 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -1,18 +1,19 @@ +import { writeFileSync, unlinkSync } from "node:fs"; import { spawn } from "node:child_process"; -import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir } from "node:fs/promises"; +import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir, symlink } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js"; import { validateAcpxRichEvent } from "../../packages/paperclip-runner/src/drivers/acpx/profile-extensions.js"; import { copilotOrigin, readCopilotToolEvidence } from "./copilot-evidence.js"; -import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js"; -import { createAttachedCommandFixture, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js"; +import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js"; +import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js"; import { runnerMatrix } from "./catalog.js"; import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/drivers/acpx/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8")); -function projected(name: string) { - const frames = fixture[name], rows: any[] = []; let clock = 10; +function projected(name: string, target = "copilot-denied-nonce.txt") { + const frames = JSON.parse(JSON.stringify(fixture[name]).replaceAll("copilot-denied-nonce.txt", target)), rows: any[] = []; let clock = 10; const projector = createCopilotToolEvidence({ sessionId: frames[0].params.sessionId, turnId: "turn", workingDirectory: "/fixture/workspace", active: () => true, emit: event => { validateAcpxRichEvent(event); rows.push({ seq: rows.length + 1, eventType: event.eventType, payload: { prpEvent: { schema: "paperclip.prp.event.v1", sourceKind: "runner", eventType: event.eventType, runId: "run", turnId: "turn", emittedAt: new Date(clock++).toISOString(), payload: event.payload } } }); } }); for (const frame of frames) { @@ -29,17 +30,17 @@ describe("Copilot Product protection integration", () => { expect(cells.find(c => c.task.id === "native-permission-deny-write")!.task.expectedTerminalState).toEqual({ issue: "in_progress", run: "cancelled" }); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(x => x.suite.id === "copilot-protection")).toBe(false); }); - it("feeds actual native denied-edit wire through canonical persistence shape and the Product oracle", () => { - const { notices } = projected("deny-write"); + it.each(["copilot-denied-nonce.txt", "pc-denied-ABC123/copilot-denied-nonce.txt"])("feeds native denied-edit wire through canonical persistence and exact target oracle: %s", target => { + const { notices } = projected("deny-write", target); const request = notices.find(n => n.stage === "permission_requested")!, delivered = notices.find(n => n.stage === "permission_delivered")!, failed = notices.find(n => n.status === "failed")!; - const e = { expected: copilotOrigin(request), requestId: "permission", expectedRelativePath: "copilot-denied-nonce.txt", + const e = { expected: copilotOrigin(request), requestId: "permission", expectedRelativePath: target, request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] }, decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" }, deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! }, toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, cancellation: { requestedAtMs: 40, scope: "run", acknowledged: true }, cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1, fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } }; - expect(request.target).toBe("copilot-denied-nonce.txt"); + expect(request.target).toBe(target); expect(gradeCopilotDeniedWrite(e).passed).toBe(true); e.request.targetRelativePath = "foreign.txt"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); e.request.targetRelativePath = e.expectedRelativePath; e.cancellation.acknowledged = false; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); @@ -65,6 +66,43 @@ describe("Copilot Product protection integration", () => { ]) { const { rows } = projected("attached-shell"); mutate(rows[0]); expect(() => readCopilotToolEvidence(rows, "run")).toThrow(); } expect(readCopilotToolEvidence([], "run")).toEqual([]); // Missing proof never fabricates an event. }); + it("isolates the denied parent from unrelated workspace startup churn and binds the prompt", async () => { + const root = await mkdtemp("/tmp/pc-denial-isolated-"); + const fixture = await createDeniedTargetFixture(root, "copilot-denied-nonce.txt"); + try { + const prompt = bindDeniedTargetPrompt(copilotProtectionCases[0].prompt("nonce"), "copilot-denied-nonce.txt", fixture.targetRelativePath); + expect(prompt).toContain(`creating ${fixture.targetRelativePath} with DENIED-nonce`); + expect(fixture.targetRelativePath).toMatch(/^pc-denied-[a-zA-Z0-9]+\/copilot-denied-nonce\.txt$/); + await mkdir(join(root, "startup")); await writeFile(join(root, "startup", "runtime.json"), "{}"); + await writeFile(join(root, "transient"), "x"); await unlink(join(root, "transient")); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(fixture.watcher.finish()).toMatchObject({ complete: true, targetMutationCount: 0, reasons: [] }); + } finally { fixture.watcher.finish(); await rm(root, { recursive: true, force: true }); } + }); + it("retains a coverage gap when create/delete occurs before callbacks can arrive", async () => { + const root = await mkdtemp("/tmp/pc-denial-gap-"); + const fixture = await createDeniedTargetFixture(root, "denied"); + try { + writeFileSync(fixture.targetPath, "x"); unlinkSync(fixture.targetPath); + const receipt = fixture.watcher.finish(); + expect(receipt).toMatchObject({ complete: false, targetMutationCount: 0 }); + expect(receipt.reasons).toContain("coverage-gap-parent-version-changed"); + expect(receipt.finalParent).not.toEqual(receipt.initialParent); + expect(fixture.watcher.finish()).toBe(receipt); + } finally { fixture.watcher.finish(); await rm(root, { recursive: true, force: true }); } + }); + it("refuses a preexisting target, symlink parent and ambiguous prompt", async () => { + const root = await mkdtemp("/tmp/pc-denial-invalid-"); + try { + await writeFile(join(root, "denied"), "present"); + expect(() => watchDeniedTarget(root, "denied")).toThrow(/initially be absent/); + await symlink(root, join(root, "link")); + expect(() => watchDeniedTarget(join(root, "link"), "absent")).toThrow(/real directory/); + expect(() => watchDeniedTarget(root, "../denied")).toThrow(/Invalid/); + expect(() => bindDeniedTargetPrompt("no target", "denied", "pc-denied-a/denied")).toThrow(/exact target once/); + expect(() => bindDeniedTargetPrompt("denied and denied", "denied", "pc-denied-a/denied")).toThrow(/exact target once/); + } finally { await rm(root, { recursive: true, force: true }); } + }); it("observes a transient create/delete even when final stat is absent", async () => { const root = await mkdtemp("/tmp/pc-copilot-watch-"); const watcher = watchDeniedTarget(root, "denied"); try { await writeFile(join(root, "denied"), "x"); await unlink(join(root, "denied")); await new Promise(r => setTimeout(r, 30)); const proof = watcher.finish(); expect(proof.targetMutationCount > 0 || !proof.complete).toBe(true); expect(await exists(join(root, "denied"))).toBe(false); } @@ -73,10 +111,10 @@ describe("Copilot Product protection integration", () => { it("rejects replacement or disappearance of the watched parent directory", async () => { const base = await mkdtemp("/tmp/pc-copilot-parent-"); const root = join(base, "workspace"); await mkdir(root); const watcher = watchDeniedTarget(root, "denied"); - try { await rename(root, join(base, "old")); await mkdir(root); expect(watcher.finish().complete).toBe(false); } + try { await rename(root, join(base, "old")); await mkdir(root); expect(watcher.finish().complete).toBe(false); expect(watcher.finish().reasons).toContain("parent-identity-changed"); } finally { watcher.finish(); await rm(base, { recursive: true, force: true }); } const gone = await mkdtemp("/tmp/pc-copilot-parent-"); const deleted = watchDeniedTarget(gone, "denied"); - await rm(gone, { recursive: true }); expect(deleted.finish().complete).toBe(false); + await rm(gone, { recursive: true }); expect(deleted.finish().complete).toBe(false); expect(deleted.finish().reasons).toContain("parent-unavailable-at-finish"); }); it("binds cleanup to the exact per-turn runner PID/start/run, never a warm or reused process", () => { const startedAt = new Date(1_700_000_000_000).toISOString(); diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 83129531ca..2c1dde732e 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -6,7 +6,7 @@ import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; -import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; +import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; @@ -39,7 +39,9 @@ export async function runCopilotProtectionFlow(input: { assertCopilotRemoteRetirement(sealed, baseline); processes = sealed.processes; return sealed; } - const target = `copilot-denied-${nonce}.txt`, targetPath = join(workspacePath, target); + const targetName = `copilot-denied-${nonce}.txt`; + const localTarget = deny && !remote ? await createDeniedTargetFixture(workspacePath, targetName) : undefined; + const target = localTarget?.targetRelativePath ?? targetName, targetPath = localTarget?.targetPath ?? join(workspacePath, target); const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = []; const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => { if (remote) { @@ -48,11 +50,11 @@ export async function runCopilotProtectionFlow(input: { remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase)); } else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) }); }; - const watcher = deny && !remote ? watchDeniedTarget(workspacePath, target) : undefined; + const watcher = localTarget?.watcher; const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`); const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath); const exactCommand = () => remoteCommand ?? command; - let watchReceipt: ReturnType["finish"]> | undefined; + let watchReceipt: CopilotDeniedWriteEvidence["mutationObservation"] | undefined; const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); }; async function load() { if (issue.id) issue = await api.get(`/api/issues/${issue.id}`); @@ -77,7 +79,7 @@ export async function runCopilotProtectionFlow(input: { workspace: { name: "Primary", sourceType: "local_path", cwd: workspacePath, isPrimary: true }, }); if (deny && !remote) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); } - const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`; + const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${localTarget ? bindDeniedTargetPrompt(execution.task.buildPrompt(nonce), targetName, target) : execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`; await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title: execution.task.buildTitle(nonce), prompt, workMode: "standard", projectName: project.name }); const found = await pollUntil({ label: "browser-created Copilot protection task", deadlineAt: input.deadlineAt, load: async () => (await api.get(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(r => r.title === execution.task.buildTitle(nonce)), accept: Boolean }); if (!found) throw new Error("Browser-created task was not found"); issue = found; diff --git a/tests/runner-e2e/cursor-native-flow.ts b/tests/runner-e2e/cursor-native-flow.ts index 10c75f9127..4a3fe14eb4 100644 --- a/tests/runner-e2e/cursor-native-flow.ts +++ b/tests/runner-e2e/cursor-native-flow.ts @@ -7,7 +7,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; -import { observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; +import { observeRunProcesses, createDeniedTargetFixture } from "./copilot-local-fixtures.js"; import { cursorDeniedCommand, hasCursorDeniedCommand, hasCursorCancellation, readCursorToolEvidence, assertCursorRemoteSnapshot, cursorRemoteDeniedSample, hasCursorRemoteRetirement, hasCursorRemoteWorkspaceUnchanged, type CursorRemoteSnapshot, type CursorRemoteBinding, type CursorToolNotice } from "./cursor-native-evidence.js"; import { cursorNativeCaseDesigns, cursorNativePlanArtifactGate, hasCursorDenialBoundary, hasCursorPlanDecision, hasDeliveredCursorNativeRequest, hasExactCursorNativeResponse, type CursorNativeMethod } from "./cursor-native-cases.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; @@ -125,7 +125,8 @@ export async function runCursorNativeFlow(input: { await input.evidence(`cursor-workspace-${phase}.json`, { baseline, current }); check(`workspace-unchanged-${phase}`, JSON.stringify(current) === JSON.stringify(baseline), "Independent workspace bytes remain unchanged by a pending/rejected/cancelled native plan or question"); }; - const deniedRelative = `cursor-denied-${nonce}.txt`; + const localDeniedTarget = !remote && design.id === "native-write-deny-reconnect" ? await createDeniedTargetFixture(input.workspacePath, `cursor-denied-${nonce}.txt`) : null; + const deniedRelative = localDeniedTarget?.targetRelativePath ?? `cursor-denied-${nonce}.txt`; let deniedPath = remote ? "" : join(input.workspacePath, deniedRelative); let deniedCommand = remote ? null : cursorDeniedCommand(deniedPath); let denialNotices: CursorToolNotice[] = []; let denialRunEvents: Row[] = []; let denialTurnId = ""; let cancelRequestedAt = NaN; let cancellationProven = false; @@ -152,7 +153,7 @@ export async function runCursorNativeFlow(input: { let processes = observeProcesses(); let deniedRequest: Row | null = null; let processTimer: ReturnType | undefined; - const watch = !remote && design.id === "native-write-deny-reconnect" ? watchDeniedTarget(input.workspacePath, `cursor-denied-${nonce}.txt`) : null; + const watch = localDeniedTarget?.watcher ?? null; if (watch) { processTimer = setInterval(() => { try { processes = observeProcesses(); } catch { processObservationError = true; } }, 250); input.registerCleanupAssertion!(async () => { diff --git a/tests/runner-e2e/denied-target-isolation-flow.test.ts b/tests/runner-e2e/denied-target-isolation-flow.test.ts new file mode 100644 index 0000000000..0f6b5f3c83 --- /dev/null +++ b/tests/runner-e2e/denied-target-isolation-flow.test.ts @@ -0,0 +1,44 @@ +import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { runCopilotProtectionFlow } from "./copilot-protection-flow.js"; +import { runCursorNativeFlow } from "./cursor-native-flow.js"; +import { cursorDeniedCommand } from "./cursor-native-evidence.js"; + +const dispatch = vi.hoisted(() => ({ inspect: async (_input: { prompt: string }) => {} })); +vi.mock("./user-actions.js", () => ({ createTaskThroughUi: async (input: { prompt: string }) => { + await dispatch.inspect(input); throw new Error("stop-before-provider-dispatch"); +} })); +vi.mock("@playwright/test", () => ({ expect: (actual: unknown, message?: string) => ({ toBe: (expected: unknown) => expect(actual, message).toBe(expected) }) })); + +it.each(["copilot", "cursor"] as const)("arms an isolated %s target before dispatch and binds the exact native operation", async candidate => { + const workspacePath = await mkdtemp(join(tmpdir(), "denial-dispatch-")); + const taskId = candidate === "copilot" ? "native-permission-deny-write" : "native-write-deny-reconnect"; + const execution = runnerMatrix.find(row => row.profile.qualificationCandidate === candidate && row.environment.id === "local" && row.task.id === taskId)!; + const receipts = new Map(); const cleanup: Array<() => Promise> = []; + let target: string | undefined; + dispatch.inspect = async ({ prompt }) => { + const parents = (await readdir(workspacePath)).filter(name => name.startsWith("pc-denied-")); + expect(parents).toHaveLength(1); + expect(await readdir(join(workspacePath, parents[0]!))).toEqual([]); + target = `${parents[0]}/${candidate}-denied-nonce.txt`; + if (candidate === "copilot") expect(prompt).toContain(`creating ${target} with DENIED-nonce`); + else expect(prompt).toContain(cursorDeniedCommand(join(workspacePath, target)).command); + await writeFile(join(workspacePath, "unrelated-startup-file"), "runtime setup"); + }; + const api = { get: async (path: string) => { + if (path === "/api/agents/agent") return { adapterConfig: { lifecycleMode: "per_turn" } }; + throw new Error(`Unexpected API call ${path}`); + }, patch: async (_path: string, value: unknown) => value, post: async () => ({ name: "fixture project" }) }; + const input = { page: {}, api, fixtures: { company: { id: "company", issuePrefix: "FIX" }, agent: { id: "agent", name: "Fixture" }, environment: { id: "environment" } }, execution, nonce: "nonce", workspacePath, deadlineAt: Date.now() + 2000, + observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { receipts.set(name, value); }, registerCleanupAssertion: (callback: () => Promise) => cleanup.push(callback) }; + try { + await expect((candidate === "copilot" ? runCopilotProtectionFlow : runCursorNativeFlow)(input as any)).rejects.toThrow("stop-before-provider-dispatch"); + if (candidate === "cursor") await expect(cleanup[0]!()).rejects.toThrow(/cleanup proof/); + const receipt = candidate === "copilot" ? receipts.get("copilot-protection-checks.json").watchReceipt : receipts.get("cursor-native-denial-cleanup-attempt.json").watcher; + expect(receipt).toMatchObject({ complete: true, targetMutationCount: 0, reasons: [] }); + expect(target).toBeDefined(); + } finally { await rm(workspacePath, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index 4c4537fb0d..eadfda593d 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -1,4 +1,4 @@ -import { observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; +import { observeRunProcesses, createDeniedTargetFixture, bindDeniedTargetPrompt } from "./copilot-local-fixtures.js"; import { hasDeliveredPiDenial, piPermissionRequests, hasPiRemoteRetirement, hasUnchangedPiRemoteTarget } from "./pi-native-evidence.js"; import { randomBytes } from "node:crypto"; import { lstat, readFile } from "node:fs/promises"; @@ -151,8 +151,9 @@ export async function runPiNativeFlow(input: { const agent = await api.get(`/api/agents/${fixtures.agent.id}`); const configured = await api.patch(`/api/agents/${fixtures.agent.id}`, { adapterConfig: { ...agent.adapterConfig, acpxPermissionMode: "approve-reads", lifecycleMode: "per_turn", timeoutSec: 120 } }); check("human-denial-policy", configured.adapterConfig.acpxPermissionMode === "approve-reads" && configured.adapterConfig.lifecycleMode === "per_turn", "Native write requires a browser permission decision in an owned per-turn process"); - const target = "pi-human-denied.txt", path = join(input.workspacePath, target); - const watcher = remote ? null : watchDeniedTarget(input.workspacePath, target), observer = remote ? null : observeRunProcesses(); + const localTarget = remote ? null : await createDeniedTargetFixture(input.workspacePath, "pi-human-denied.txt"); + const target = localTarget?.targetRelativePath ?? "pi-human-denied.txt", path = join(input.workspacePath, target); + const watcher = localTarget?.watcher ?? null, observer = remote ? null : observeRunProcesses(); let processError = false, processAuthority: string | undefined; const observe = () => { const run = runs[0]; const authority = run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined; @@ -170,10 +171,10 @@ export async function runPiNativeFlow(input: { await input.evidence("pi-human-denial-retirement.json", { processes, processError, fileAbsent, journal, passed }); if (!passed) throw new Error("Pi human denial lacks independent no-effect and provider-retirement proof"); return [{ id: "human-denial-through-retirement", passed, detail: "Browser-denied target stayed absent through exact owned provider-process retirement" }]; - } finally { clearInterval(timer); watcher!.finish(); } + } finally { clearInterval(timer); await input.evidence("pi-human-denial-cleanup-attempt.json", { target, processes, processError, journal: watcher!.finish() }); } }); if (!remote) check("human-target-initially-absent", await absent(path), "Independent target is absent before provider work"); - await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: [target] }); + await create(execution.task.buildTitle(nonce), localTarget ? bindDeniedTargetPrompt(execution.task.buildPrompt(nonce), "pi-human-denied.txt", target) : execution.task.buildPrompt(nonce), { targets: [target] }); if (remote) check("human-target-initially-absent", currentBaseline?.targets[target]?.absent === true && currentBaseline.targets[target]!.complete, "Remote watcher was armed before action publication with an absent target"); const pending = await pollUntil({ label: "Pi native browser permission", deadlineAt: input.deadlineAt, load: async () => { const state = await load(); processes = observe(); return { ...state, events: state.runs.length === 1 ? await events(state.runs[0]!.id) : [] }; }, reject: rejectFailure, accept: state => state.runs.length === 1 && piPermissionRequests(state.events, state.runs[0]!.id).length === 1 }); diff --git a/tests/runner-e2e/pi-native-human-flow.test.ts b/tests/runner-e2e/pi-native-human-flow.test.ts index 86a3dd85d0..3d6f1c24fe 100644 --- a/tests/runner-e2e/pi-native-human-flow.test.ts +++ b/tests/runner-e2e/pi-native-human-flow.test.ts @@ -4,25 +4,30 @@ import { join } from "node:path"; import { expect, it, vi } from "vitest"; import { piNativeTasks } from "./pi-native-cases.js"; import { runPiNativeFlow } from "./pi-native-flow.js"; -const proof = vi.hoisted(() => ({ mutation: false, live: false })); -vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn() })); -vi.mock("./copilot-local-fixtures.js", () => ({ - watchDeniedTarget: () => ({ finish: () => ({ complete: true, targetMutationCount: proof.mutation ? 1 : 0 }) }), +const proof = vi.hoisted(() => ({ mutation: false, incomplete: false, live: false, target: "pi-human-denied.txt", prompt: "" })); +vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn(async (input: { prompt: string }) => { proof.prompt = input.prompt; }) })); +vi.mock("./copilot-local-fixtures.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, + createDeniedTargetFixture: async (workspace: string, name: string) => { + const fixture = await actual.createDeniedTargetFixture(workspace, name); proof.target = fixture.targetRelativePath; + return { ...fixture, watcher: { finish: () => ({ ...fixture.watcher.finish(), complete: !proof.incomplete && fixture.watcher.finish().complete, targetMutationCount: proof.mutation ? 1 : 0 }) } }; + }, observeRunProcesses: () => ({ sample: () => ({ captured: true, live: proof.live ? [123] : [], journal: [] }) }), -})); +}; }); vi.mock("@playwright/test", () => ({ expect: (actual: any, message?: string) => ({ toBe: (value: unknown) => expect(actual, message).toBe(value), toBeVisible: async () => {}, toHaveCount: async (value: number) => expect(actual.count).toBe(value), }) })); const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload } } }); -async function exercise(mutation: boolean, remote = false, adapter = "acpx-runtime") { - proof.mutation = mutation; proof.live = false; +async function exercise(mutation: boolean, remote = false, adapter = "acpx-runtime", incomplete = false) { + proof.mutation = mutation; proof.incomplete = incomplete; proof.live = false; proof.target = "pi-human-denied.txt"; proof.prompt = ""; const workspacePath = await mkdtemp(join(tmpdir(), "pi-human-fixture-")); let declined = false, browserPosts = 0; const saved = new Map(); const cleanup: Array<() => Promise> = []; const task = piNativeTasks.find(row => row.id === "human-permission-denial")!; const issue = () => ({ id: "issue", identifier: "PI-1", title: task.buildTitle("fixture"), status: declined ? "done" : "in_progress" }); const run = () => ({ id: "run", status: declined ? "succeeded" : "running", runtimeMode: "native", processPid: 123, processGroupId: 123, processStartedAt: "2026-09-29T00:00:00Z" }); const request = { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" }, origin: { adapter, provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline", label: "Decline" }] }; - const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: "pi-human-denied.txt", status: "failed", output: "Pi operation was denied or cancelled" })] : [])]; + const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: proof.target, status: "failed", output: "Pi operation was denied or cancelled" })] : [])]; const api = { post: async () => ({ name: "Pi fixture project" }), patch: async (_path: string, value: any) => value, get: async (path: string) => { @@ -47,11 +52,12 @@ async function exercise(mutation: boolean, remote = false, adapter = "acpx-runti try { if (remote) await writeFile(join(workspacePath, "pi-human-denied.txt"), "POISON HOST COPYBACK"); const result = await runPiNativeFlow({ page, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "environment" } }, execution: { task, environment: { id: remote ? "daytona" : "local" }, profile: { qualificationCandidate: "pi" } }, workspacePath, nonce: "fixture", deadlineAt: Date.now() + 2000, restart: async () => {}, observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { saved.set(name, value); }, remoteBootstrap, registerCleanupAssertion: (fn: () => Promise) => cleanup.push(fn) } as any); + if (!remote) { expect(proof.target).toMatch(/^pc-denied-[a-zA-Z0-9]+\/pi-human-denied\.txt$/); expect(proof.prompt).toContain(`relative path ${proof.target} and content forbidden`); } expect(browserPosts).toBe(1); expect(result.checks.every(check => check.passed)).toBe(true); expect(saved.has("api-state.json")).toBe(true); expect(cleanup).toHaveLength(1); - if (mutation) await expect(cleanup[0]!()).rejects.toThrow("no-effect"); + if (mutation || incomplete) await expect(cleanup[0]!()).rejects.toThrow("no-effect"); else expect((await cleanup[0]!())[0].passed).toBe(true); - expect(saved.get(remote ? "pi-remote-1-retirement.json" : "pi-human-denial-retirement.json").passed).toBe(!mutation); + expect(saved.get(remote ? "pi-remote-1-retirement.json" : "pi-human-denial-retirement.json").passed).toBe(!mutation && !incomplete); } finally { await rm(workspacePath, { recursive: true, force: true }); } } it("drives actual browser-denial flow and independent retirement proof", async () => exercise(false)); @@ -60,3 +66,5 @@ it("rejects an observed create/delete even when final target is absent", async ( it("proves browser denial against remote watcher and retirement without trusting a host file", async () => exercise(false, true)); it.each([false, true])("drives sidecar permission denial with remote=%s", async remote => exercise(false, remote, "acpx-runtime-sidecar")); + +it("rejects incomplete local observation even with zero target mutations", async () => exercise(false, false, "acpx-runtime", true));