mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-09 06:15:21 +02:00
test(runner-e2e): isolate denied-target watchers from startup churn
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
632e7c9802
commit
d6ff17fd0d
9 files changed
+190
-46
No files matched your search
@@ -3,31 +3,71 @@ import { createHash, randomBytes } from "node:crypto";
|
||||
import { watch, lstatSync, type FSWatcher } from "node:fs";
|
||||
import { lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer, type Socket } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { basename, join, relative } from "node:path";
|
||||
|
||||
export const sha256 = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
export async function exists(path: string): Promise<boolean> {
|
||||
try { await lstat(path); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; }
|
||||
}
|
||||
/** Allocate before dispatch; ordinary workspace startup cannot mutate this parent. */
|
||||
export async function createDeniedTargetFixture(workspacePath: string, name: string) {
|
||||
if (!name || basename(name) !== name || name === "." || name === "..") throw new Error("Invalid denied target name");
|
||||
const directory = await mkdtemp(join(workspacePath, "pc-denied-"));
|
||||
const targetPath = join(directory, name);
|
||||
return { directory, targetPath, targetRelativePath: relative(workspacePath, targetPath), watcher: watchDeniedTarget(directory, name) };
|
||||
}
|
||||
|
||||
/** Substitute the one authored target, never append a contradictory second path. */
|
||||
export function bindDeniedTargetPrompt(prompt: string, original: string, target: string): string {
|
||||
const parts = prompt.split(original);
|
||||
if (parts.length !== 2) throw new Error("Denied prompt must name its exact target once");
|
||||
return parts.join(target);
|
||||
}
|
||||
|
||||
export function watchDeniedTarget(directory: string, name: string) {
|
||||
const startedAtMs = Date.now(); let complete = true, targetMutationCount = 0;
|
||||
if (!name || basename(name) !== name || name === "." || name === "..") throw new Error("Invalid denied target name");
|
||||
const startedAtMs = Date.now(); let targetMutationCount = 0;
|
||||
const reasons = new Set<string>();
|
||||
const before = lstatSync(directory, { bigint: true });
|
||||
let final: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | undefined;
|
||||
const watcher: FSWatcher = watch(directory, (_kind, filename) => {
|
||||
if (filename === null) complete = false;
|
||||
else if (String(filename) === name) targetMutationCount++;
|
||||
if (!before.isDirectory() || before.isSymbolicLink()) throw new Error("Denied target parent must be a real directory");
|
||||
const targetAbsent = () => { try { lstatSync(join(directory, name)); return false; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; throw error; } };
|
||||
if (!targetAbsent()) throw new Error("Denied target must initially be absent");
|
||||
const identity = (stat: import("node:fs").BigIntStats) => ({ dev: String(stat.dev), ino: String(stat.ino), mtimeNs: String(stat.mtimeNs), ctimeNs: String(stat.ctimeNs) });
|
||||
const events: Array<{ sequence: number; observedAtMs: number; kind: string; target: boolean; filenameKnown: boolean }> = [];
|
||||
let eventCount = 0, lastEventAtMs = startedAtMs, closing = false;
|
||||
let final: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number; reasons: string[]; initialParent: ReturnType<typeof identity>; finalParent: ReturnType<typeof identity> | null; events: typeof events } | undefined;
|
||||
const watcher: FSWatcher = watch(directory, (kind, filename) => {
|
||||
const observedAtMs = Date.now();
|
||||
if (observedAtMs < lastEventAtMs) reasons.add("event-order-invalid");
|
||||
lastEventAtMs = observedAtMs;
|
||||
const target = filename !== null && String(filename) === name;
|
||||
if (filename === null) reasons.add("event-filename-missing");
|
||||
if (target) targetMutationCount++;
|
||||
if (++eventCount <= 128) events.push({ sequence: eventCount, observedAtMs, kind, target, filenameKnown: filename !== null });
|
||||
else reasons.add("event-journal-overflow");
|
||||
});
|
||||
watcher.on("error", () => { complete = false; });
|
||||
watcher.on("error", () => { reasons.add("watch-error"); });
|
||||
watcher.on("close", () => { if (!closing) reasons.add("watch-closed-before-finish"); });
|
||||
// Pin both identity and directory version across watcher installation.
|
||||
try {
|
||||
const armed = lstatSync(directory, { bigint: true });
|
||||
if (!armed.isDirectory() || armed.dev !== before.dev || armed.ino !== before.ino) reasons.add("parent-identity-changed-during-arm");
|
||||
if (armed.mtimeNs !== before.mtimeNs || armed.ctimeNs !== before.ctimeNs || !targetAbsent()) reasons.add("coverage-gap-during-arm");
|
||||
} catch { reasons.add("parent-unavailable-during-arm"); }
|
||||
return { finish() {
|
||||
if (final) return final;
|
||||
let after: import("node:fs").BigIntStats | undefined;
|
||||
try { after = lstatSync(directory, { bigint: true }); } catch { complete = false; }
|
||||
try { after = lstatSync(directory, { bigint: true }); } catch { reasons.add("parent-unavailable-at-finish"); }
|
||||
// FSEvents may coalesce a rapid create/delete. A changed directory version
|
||||
// with no attributed event is a coverage gap, never proof of no mutation.
|
||||
if (!after || after.dev !== before.dev || after.ino !== before.ino || !after.isDirectory()) complete = false;
|
||||
if (after && (after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) && targetMutationCount === 0) complete = false;
|
||||
final = { startedAtMs, endedAtMs: Date.now(), complete, targetMutationCount }; watcher.close(); return final;
|
||||
if (after && (after.dev !== before.dev || after.ino !== before.ino || !after.isDirectory())) reasons.add("parent-identity-changed");
|
||||
if (after && (after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) && targetMutationCount === 0) reasons.add("coverage-gap-parent-version-changed");
|
||||
try { if (!targetAbsent() && targetMutationCount === 0) reasons.add("coverage-gap-unobserved-target"); } catch { reasons.add("target-unavailable-at-finish"); }
|
||||
const endedAtMs = Date.now();
|
||||
if (endedAtMs < lastEventAtMs) reasons.add("event-order-invalid");
|
||||
final = { startedAtMs, endedAtMs, complete: reasons.size === 0, targetMutationCount, reasons: [...reasons], initialParent: identity(before), finalParent: after ? identity(after) : null, events };
|
||||
closing = true; watcher.close(); return final;
|
||||
} };
|
||||
}
|
||||
interface ProcessIdentity { pid: number; parent: number; start: string }
|
||||
|
||||
@@ -37,6 +37,16 @@ describe("Copilot protection Product oracles", () => {
|
||||
it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => {
|
||||
expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] });
|
||||
});
|
||||
it("requires the exact isolated target in native permission evidence", () => {
|
||||
const e = denied(); e.expectedRelativePath = "pc-denied-ABC123/copilot-denied-nonce.txt";
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
e.request!.targetRelativePath = e.expectedRelativePath;
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
|
||||
for (const path of ["../copilot-denied-nonce.txt", "other/copilot-denied-nonce.txt", "pc-denied-ABC123/../copilot-denied-nonce.txt"]) {
|
||||
e.expectedRelativePath = path; e.request!.targetRelativePath = path;
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => {
|
||||
const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
});
|
||||
|
||||
@@ -77,7 +77,7 @@ function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded"
|
||||
export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
|
||||
const failures: string[] = []; lifecycle(e, failures, "cancelled");
|
||||
if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation");
|
||||
if (!/^copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
|
||||
if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
|
||||
if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
|
||||
if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection");
|
||||
if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
import { writeFileSync, unlinkSync } from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir } from "node:fs/promises";
|
||||
import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir, symlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js";
|
||||
import { validateAcpxRichEvent } from "../../packages/paperclip-runner/src/drivers/acpx/profile-extensions.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence } from "./copilot-evidence.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js";
|
||||
import { createAttachedCommandFixture, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js";
|
||||
import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js";
|
||||
|
||||
const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/drivers/acpx/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"));
|
||||
function projected(name: string) {
|
||||
const frames = fixture[name], rows: any[] = []; let clock = 10;
|
||||
function projected(name: string, target = "copilot-denied-nonce.txt") {
|
||||
const frames = JSON.parse(JSON.stringify(fixture[name]).replaceAll("copilot-denied-nonce.txt", target)), rows: any[] = []; let clock = 10;
|
||||
const projector = createCopilotToolEvidence({ sessionId: frames[0].params.sessionId, turnId: "turn", workingDirectory: "/fixture/workspace", active: () => true,
|
||||
emit: event => { validateAcpxRichEvent(event); rows.push({ seq: rows.length + 1, eventType: event.eventType, payload: { prpEvent: { schema: "paperclip.prp.event.v1", sourceKind: "runner", eventType: event.eventType, runId: "run", turnId: "turn", emittedAt: new Date(clock++).toISOString(), payload: event.payload } } }); } });
|
||||
for (const frame of frames) {
|
||||
@@ -29,17 +30,17 @@ describe("Copilot Product protection integration", () => {
|
||||
expect(cells.find(c => c.task.id === "native-permission-deny-write")!.task.expectedTerminalState).toEqual({ issue: "in_progress", run: "cancelled" });
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(x => x.suite.id === "copilot-protection")).toBe(false);
|
||||
});
|
||||
it("feeds actual native denied-edit wire through canonical persistence shape and the Product oracle", () => {
|
||||
const { notices } = projected("deny-write");
|
||||
it.each(["copilot-denied-nonce.txt", "pc-denied-ABC123/copilot-denied-nonce.txt"])("feeds native denied-edit wire through canonical persistence and exact target oracle: %s", target => {
|
||||
const { notices } = projected("deny-write", target);
|
||||
const request = notices.find(n => n.stage === "permission_requested")!, delivered = notices.find(n => n.stage === "permission_delivered")!, failed = notices.find(n => n.status === "failed")!;
|
||||
const e = { expected: copilotOrigin(request), requestId: "permission", expectedRelativePath: "copilot-denied-nonce.txt",
|
||||
const e = { expected: copilotOrigin(request), requestId: "permission", expectedRelativePath: target,
|
||||
request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] },
|
||||
decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" },
|
||||
deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! },
|
||||
toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, cancellation: { requestedAtMs: 40, scope: "run", acknowledged: true },
|
||||
cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1,
|
||||
fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } };
|
||||
expect(request.target).toBe("copilot-denied-nonce.txt");
|
||||
expect(request.target).toBe(target);
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
|
||||
e.request.targetRelativePath = "foreign.txt"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
e.request.targetRelativePath = e.expectedRelativePath; e.cancellation.acknowledged = false; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
@@ -65,6 +66,43 @@ describe("Copilot Product protection integration", () => {
|
||||
]) { const { rows } = projected("attached-shell"); mutate(rows[0]); expect(() => readCopilotToolEvidence(rows, "run")).toThrow(); }
|
||||
expect(readCopilotToolEvidence([], "run")).toEqual([]); // Missing proof never fabricates an event.
|
||||
});
|
||||
it("isolates the denied parent from unrelated workspace startup churn and binds the prompt", async () => {
|
||||
const root = await mkdtemp("/tmp/pc-denial-isolated-");
|
||||
const fixture = await createDeniedTargetFixture(root, "copilot-denied-nonce.txt");
|
||||
try {
|
||||
const prompt = bindDeniedTargetPrompt(copilotProtectionCases[0].prompt("nonce"), "copilot-denied-nonce.txt", fixture.targetRelativePath);
|
||||
expect(prompt).toContain(`creating ${fixture.targetRelativePath} with DENIED-nonce`);
|
||||
expect(fixture.targetRelativePath).toMatch(/^pc-denied-[a-zA-Z0-9]+\/copilot-denied-nonce\.txt$/);
|
||||
await mkdir(join(root, "startup")); await writeFile(join(root, "startup", "runtime.json"), "{}");
|
||||
await writeFile(join(root, "transient"), "x"); await unlink(join(root, "transient"));
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
expect(fixture.watcher.finish()).toMatchObject({ complete: true, targetMutationCount: 0, reasons: [] });
|
||||
} finally { fixture.watcher.finish(); await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
it("retains a coverage gap when create/delete occurs before callbacks can arrive", async () => {
|
||||
const root = await mkdtemp("/tmp/pc-denial-gap-");
|
||||
const fixture = await createDeniedTargetFixture(root, "denied");
|
||||
try {
|
||||
writeFileSync(fixture.targetPath, "x"); unlinkSync(fixture.targetPath);
|
||||
const receipt = fixture.watcher.finish();
|
||||
expect(receipt).toMatchObject({ complete: false, targetMutationCount: 0 });
|
||||
expect(receipt.reasons).toContain("coverage-gap-parent-version-changed");
|
||||
expect(receipt.finalParent).not.toEqual(receipt.initialParent);
|
||||
expect(fixture.watcher.finish()).toBe(receipt);
|
||||
} finally { fixture.watcher.finish(); await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
it("refuses a preexisting target, symlink parent and ambiguous prompt", async () => {
|
||||
const root = await mkdtemp("/tmp/pc-denial-invalid-");
|
||||
try {
|
||||
await writeFile(join(root, "denied"), "present");
|
||||
expect(() => watchDeniedTarget(root, "denied")).toThrow(/initially be absent/);
|
||||
await symlink(root, join(root, "link"));
|
||||
expect(() => watchDeniedTarget(join(root, "link"), "absent")).toThrow(/real directory/);
|
||||
expect(() => watchDeniedTarget(root, "../denied")).toThrow(/Invalid/);
|
||||
expect(() => bindDeniedTargetPrompt("no target", "denied", "pc-denied-a/denied")).toThrow(/exact target once/);
|
||||
expect(() => bindDeniedTargetPrompt("denied and denied", "denied", "pc-denied-a/denied")).toThrow(/exact target once/);
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
it("observes a transient create/delete even when final stat is absent", async () => {
|
||||
const root = await mkdtemp("/tmp/pc-copilot-watch-"); const watcher = watchDeniedTarget(root, "denied");
|
||||
try { await writeFile(join(root, "denied"), "x"); await unlink(join(root, "denied")); await new Promise(r => setTimeout(r, 30)); const proof = watcher.finish(); expect(proof.targetMutationCount > 0 || !proof.complete).toBe(true); expect(await exists(join(root, "denied"))).toBe(false); }
|
||||
@@ -73,10 +111,10 @@ describe("Copilot Product protection integration", () => {
|
||||
it("rejects replacement or disappearance of the watched parent directory", async () => {
|
||||
const base = await mkdtemp("/tmp/pc-copilot-parent-"); const root = join(base, "workspace"); await mkdir(root);
|
||||
const watcher = watchDeniedTarget(root, "denied");
|
||||
try { await rename(root, join(base, "old")); await mkdir(root); expect(watcher.finish().complete).toBe(false); }
|
||||
try { await rename(root, join(base, "old")); await mkdir(root); expect(watcher.finish().complete).toBe(false); expect(watcher.finish().reasons).toContain("parent-identity-changed"); }
|
||||
finally { watcher.finish(); await rm(base, { recursive: true, force: true }); }
|
||||
const gone = await mkdtemp("/tmp/pc-copilot-parent-"); const deleted = watchDeniedTarget(gone, "denied");
|
||||
await rm(gone, { recursive: true }); expect(deleted.finish().complete).toBe(false);
|
||||
await rm(gone, { recursive: true }); expect(deleted.finish().complete).toBe(false); expect(deleted.finish().reasons).toContain("parent-unavailable-at-finish");
|
||||
});
|
||||
it("binds cleanup to the exact per-turn runner PID/start/run, never a warm or reused process", () => {
|
||||
const startedAt = new Date(1_700_000_000_000).toISOString();
|
||||
|
||||
@@ -6,7 +6,7 @@ import { pollUntil, type RunnerApi } from "./api.js";
|
||||
import { collectRunEvents } from "./run-observations.js";
|
||||
import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
|
||||
import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
@@ -39,7 +39,9 @@ export async function runCopilotProtectionFlow(input: {
|
||||
assertCopilotRemoteRetirement(sealed, baseline); processes = sealed.processes;
|
||||
return sealed;
|
||||
}
|
||||
const target = `copilot-denied-${nonce}.txt`, targetPath = join(workspacePath, target);
|
||||
const targetName = `copilot-denied-${nonce}.txt`;
|
||||
const localTarget = deny && !remote ? await createDeniedTargetFixture(workspacePath, targetName) : undefined;
|
||||
const target = localTarget?.targetRelativePath ?? targetName, targetPath = localTarget?.targetPath ?? join(workspacePath, target);
|
||||
const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = [];
|
||||
const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => {
|
||||
if (remote) {
|
||||
@@ -48,11 +50,11 @@ export async function runCopilotProtectionFlow(input: {
|
||||
remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase));
|
||||
} else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) });
|
||||
};
|
||||
const watcher = deny && !remote ? watchDeniedTarget(workspacePath, target) : undefined;
|
||||
const watcher = localTarget?.watcher;
|
||||
const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`);
|
||||
const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath);
|
||||
const exactCommand = () => remoteCommand ?? command;
|
||||
let watchReceipt: ReturnType<ReturnType<typeof watchDeniedTarget>["finish"]> | undefined;
|
||||
let watchReceipt: CopilotDeniedWriteEvidence["mutationObservation"] | undefined;
|
||||
const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); };
|
||||
async function load() {
|
||||
if (issue.id) issue = await api.get<Row>(`/api/issues/${issue.id}`);
|
||||
@@ -77,7 +79,7 @@ export async function runCopilotProtectionFlow(input: {
|
||||
workspace: { name: "Primary", sourceType: "local_path", cwd: workspacePath, isPrimary: true },
|
||||
});
|
||||
if (deny && !remote) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); }
|
||||
const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`;
|
||||
const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${localTarget ? bindDeniedTargetPrompt(execution.task.buildPrompt(nonce), targetName, target) : execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`;
|
||||
await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title: execution.task.buildTitle(nonce), prompt, workMode: "standard", projectName: project.name });
|
||||
const found = await pollUntil({ label: "browser-created Copilot protection task", deadlineAt: input.deadlineAt, load: async () => (await api.get<Row[]>(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(r => r.title === execution.task.buildTitle(nonce)), accept: Boolean });
|
||||
if (!found) throw new Error("Browser-created task was not found"); issue = found;
|
||||
|
||||
@@ -7,7 +7,7 @@ import { expect, type Page } from "@playwright/test";
|
||||
import { pollUntil, type RunnerApi } from "./api.js";
|
||||
import { collectRunEvents } from "./run-observations.js";
|
||||
import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
|
||||
import { observeRunProcesses, createDeniedTargetFixture } from "./copilot-local-fixtures.js";
|
||||
import { cursorDeniedCommand, hasCursorDeniedCommand, hasCursorCancellation, readCursorToolEvidence, assertCursorRemoteSnapshot, cursorRemoteDeniedSample, hasCursorRemoteRetirement, hasCursorRemoteWorkspaceUnchanged, type CursorRemoteSnapshot, type CursorRemoteBinding, type CursorToolNotice } from "./cursor-native-evidence.js";
|
||||
import { cursorNativeCaseDesigns, cursorNativePlanArtifactGate, hasCursorDenialBoundary, hasCursorPlanDecision, hasDeliveredCursorNativeRequest, hasExactCursorNativeResponse, type CursorNativeMethod } from "./cursor-native-cases.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
@@ -125,7 +125,8 @@ export async function runCursorNativeFlow(input: {
|
||||
await input.evidence(`cursor-workspace-${phase}.json`, { baseline, current });
|
||||
check(`workspace-unchanged-${phase}`, JSON.stringify(current) === JSON.stringify(baseline), "Independent workspace bytes remain unchanged by a pending/rejected/cancelled native plan or question");
|
||||
};
|
||||
const deniedRelative = `cursor-denied-${nonce}.txt`;
|
||||
const localDeniedTarget = !remote && design.id === "native-write-deny-reconnect" ? await createDeniedTargetFixture(input.workspacePath, `cursor-denied-${nonce}.txt`) : null;
|
||||
const deniedRelative = localDeniedTarget?.targetRelativePath ?? `cursor-denied-${nonce}.txt`;
|
||||
let deniedPath = remote ? "" : join(input.workspacePath, deniedRelative);
|
||||
let deniedCommand = remote ? null : cursorDeniedCommand(deniedPath);
|
||||
let denialNotices: CursorToolNotice[] = []; let denialRunEvents: Row[] = []; let denialTurnId = ""; let cancelRequestedAt = NaN; let cancellationProven = false;
|
||||
@@ -152,7 +153,7 @@ export async function runCursorNativeFlow(input: {
|
||||
let processes = observeProcesses();
|
||||
let deniedRequest: Row | null = null;
|
||||
let processTimer: ReturnType<typeof setInterval> | undefined;
|
||||
const watch = !remote && design.id === "native-write-deny-reconnect" ? watchDeniedTarget(input.workspacePath, `cursor-denied-${nonce}.txt`) : null;
|
||||
const watch = localDeniedTarget?.watcher ?? null;
|
||||
if (watch) {
|
||||
processTimer = setInterval(() => { try { processes = observeProcesses(); } catch { processObservationError = true; } }, 250);
|
||||
input.registerCleanupAssertion!(async () => {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { runCopilotProtectionFlow } from "./copilot-protection-flow.js";
|
||||
import { runCursorNativeFlow } from "./cursor-native-flow.js";
|
||||
import { cursorDeniedCommand } from "./cursor-native-evidence.js";
|
||||
|
||||
const dispatch = vi.hoisted(() => ({ inspect: async (_input: { prompt: string }) => {} }));
|
||||
vi.mock("./user-actions.js", () => ({ createTaskThroughUi: async (input: { prompt: string }) => {
|
||||
await dispatch.inspect(input); throw new Error("stop-before-provider-dispatch");
|
||||
} }));
|
||||
vi.mock("@playwright/test", () => ({ expect: (actual: unknown, message?: string) => ({ toBe: (expected: unknown) => expect(actual, message).toBe(expected) }) }));
|
||||
|
||||
it.each(["copilot", "cursor"] as const)("arms an isolated %s target before dispatch and binds the exact native operation", async candidate => {
|
||||
const workspacePath = await mkdtemp(join(tmpdir(), "denial-dispatch-"));
|
||||
const taskId = candidate === "copilot" ? "native-permission-deny-write" : "native-write-deny-reconnect";
|
||||
const execution = runnerMatrix.find(row => row.profile.qualificationCandidate === candidate && row.environment.id === "local" && row.task.id === taskId)!;
|
||||
const receipts = new Map<string, any>(); const cleanup: Array<() => Promise<unknown>> = [];
|
||||
let target: string | undefined;
|
||||
dispatch.inspect = async ({ prompt }) => {
|
||||
const parents = (await readdir(workspacePath)).filter(name => name.startsWith("pc-denied-"));
|
||||
expect(parents).toHaveLength(1);
|
||||
expect(await readdir(join(workspacePath, parents[0]!))).toEqual([]);
|
||||
target = `${parents[0]}/${candidate}-denied-nonce.txt`;
|
||||
if (candidate === "copilot") expect(prompt).toContain(`creating ${target} with DENIED-nonce`);
|
||||
else expect(prompt).toContain(cursorDeniedCommand(join(workspacePath, target)).command);
|
||||
await writeFile(join(workspacePath, "unrelated-startup-file"), "runtime setup");
|
||||
};
|
||||
const api = { get: async (path: string) => {
|
||||
if (path === "/api/agents/agent") return { adapterConfig: { lifecycleMode: "per_turn" } };
|
||||
throw new Error(`Unexpected API call ${path}`);
|
||||
}, patch: async (_path: string, value: unknown) => value, post: async () => ({ name: "fixture project" }) };
|
||||
const input = { page: {}, api, fixtures: { company: { id: "company", issuePrefix: "FIX" }, agent: { id: "agent", name: "Fixture" }, environment: { id: "environment" } }, execution, nonce: "nonce", workspacePath, deadlineAt: Date.now() + 2000,
|
||||
observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { receipts.set(name, value); }, registerCleanupAssertion: (callback: () => Promise<unknown>) => cleanup.push(callback) };
|
||||
try {
|
||||
await expect((candidate === "copilot" ? runCopilotProtectionFlow : runCursorNativeFlow)(input as any)).rejects.toThrow("stop-before-provider-dispatch");
|
||||
if (candidate === "cursor") await expect(cleanup[0]!()).rejects.toThrow(/cleanup proof/);
|
||||
const receipt = candidate === "copilot" ? receipts.get("copilot-protection-checks.json").watchReceipt : receipts.get("cursor-native-denial-cleanup-attempt.json").watcher;
|
||||
expect(receipt).toMatchObject({ complete: true, targetMutationCount: 0, reasons: [] });
|
||||
expect(target).toBeDefined();
|
||||
} finally { await rm(workspacePath, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
|
||||
import { observeRunProcesses, createDeniedTargetFixture, bindDeniedTargetPrompt } from "./copilot-local-fixtures.js";
|
||||
import { hasDeliveredPiDenial, piPermissionRequests, hasPiRemoteRetirement, hasUnchangedPiRemoteTarget } from "./pi-native-evidence.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { lstat, readFile } from "node:fs/promises";
|
||||
@@ -151,8 +151,9 @@ export async function runPiNativeFlow(input: {
|
||||
const agent = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
|
||||
const configured = await api.patch<Row>(`/api/agents/${fixtures.agent.id}`, { adapterConfig: { ...agent.adapterConfig, acpxPermissionMode: "approve-reads", lifecycleMode: "per_turn", timeoutSec: 120 } });
|
||||
check("human-denial-policy", configured.adapterConfig.acpxPermissionMode === "approve-reads" && configured.adapterConfig.lifecycleMode === "per_turn", "Native write requires a browser permission decision in an owned per-turn process");
|
||||
const target = "pi-human-denied.txt", path = join(input.workspacePath, target);
|
||||
const watcher = remote ? null : watchDeniedTarget(input.workspacePath, target), observer = remote ? null : observeRunProcesses();
|
||||
const localTarget = remote ? null : await createDeniedTargetFixture(input.workspacePath, "pi-human-denied.txt");
|
||||
const target = localTarget?.targetRelativePath ?? "pi-human-denied.txt", path = join(input.workspacePath, target);
|
||||
const watcher = localTarget?.watcher ?? null, observer = remote ? null : observeRunProcesses();
|
||||
let processError = false, processAuthority: string | undefined;
|
||||
const observe = () => {
|
||||
const run = runs[0]; const authority = run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined;
|
||||
@@ -170,10 +171,10 @@ export async function runPiNativeFlow(input: {
|
||||
await input.evidence("pi-human-denial-retirement.json", { processes, processError, fileAbsent, journal, passed });
|
||||
if (!passed) throw new Error("Pi human denial lacks independent no-effect and provider-retirement proof");
|
||||
return [{ id: "human-denial-through-retirement", passed, detail: "Browser-denied target stayed absent through exact owned provider-process retirement" }];
|
||||
} finally { clearInterval(timer); watcher!.finish(); }
|
||||
} finally { clearInterval(timer); await input.evidence("pi-human-denial-cleanup-attempt.json", { target, processes, processError, journal: watcher!.finish() }); }
|
||||
});
|
||||
if (!remote) check("human-target-initially-absent", await absent(path), "Independent target is absent before provider work");
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: [target] });
|
||||
await create(execution.task.buildTitle(nonce), localTarget ? bindDeniedTargetPrompt(execution.task.buildPrompt(nonce), "pi-human-denied.txt", target) : execution.task.buildPrompt(nonce), { targets: [target] });
|
||||
if (remote) check("human-target-initially-absent", currentBaseline?.targets[target]?.absent === true && currentBaseline.targets[target]!.complete, "Remote watcher was armed before action publication with an absent target");
|
||||
const pending = await pollUntil({ label: "Pi native browser permission", deadlineAt: input.deadlineAt, load: async () => { const state = await load(); processes = observe(); return { ...state, events: state.runs.length === 1 ? await events(state.runs[0]!.id) : [] }; }, reject: rejectFailure,
|
||||
accept: state => state.runs.length === 1 && piPermissionRequests(state.events, state.runs[0]!.id).length === 1 });
|
||||
|
||||
@@ -4,25 +4,30 @@ import { join } from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
const proof = vi.hoisted(() => ({ mutation: false, live: false }));
|
||||
vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn() }));
|
||||
vi.mock("./copilot-local-fixtures.js", () => ({
|
||||
watchDeniedTarget: () => ({ finish: () => ({ complete: true, targetMutationCount: proof.mutation ? 1 : 0 }) }),
|
||||
const proof = vi.hoisted(() => ({ mutation: false, incomplete: false, live: false, target: "pi-human-denied.txt", prompt: "" }));
|
||||
vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn(async (input: { prompt: string }) => { proof.prompt = input.prompt; }) }));
|
||||
vi.mock("./copilot-local-fixtures.js", async importOriginal => {
|
||||
const actual = await importOriginal<typeof import("./copilot-local-fixtures.js")>();
|
||||
return { ...actual,
|
||||
createDeniedTargetFixture: async (workspace: string, name: string) => {
|
||||
const fixture = await actual.createDeniedTargetFixture(workspace, name); proof.target = fixture.targetRelativePath;
|
||||
return { ...fixture, watcher: { finish: () => ({ ...fixture.watcher.finish(), complete: !proof.incomplete && fixture.watcher.finish().complete, targetMutationCount: proof.mutation ? 1 : 0 }) } };
|
||||
},
|
||||
observeRunProcesses: () => ({ sample: () => ({ captured: true, live: proof.live ? [123] : [], journal: [] }) }),
|
||||
}));
|
||||
}; });
|
||||
vi.mock("@playwright/test", () => ({ expect: (actual: any, message?: string) => ({
|
||||
toBe: (value: unknown) => expect(actual, message).toBe(value), toBeVisible: async () => {}, toHaveCount: async (value: number) => expect(actual.count).toBe(value),
|
||||
}) }));
|
||||
const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload } } });
|
||||
async function exercise(mutation: boolean, remote = false, adapter = "acpx-runtime") {
|
||||
proof.mutation = mutation; proof.live = false;
|
||||
async function exercise(mutation: boolean, remote = false, adapter = "acpx-runtime", incomplete = false) {
|
||||
proof.mutation = mutation; proof.incomplete = incomplete; proof.live = false; proof.target = "pi-human-denied.txt"; proof.prompt = "";
|
||||
const workspacePath = await mkdtemp(join(tmpdir(), "pi-human-fixture-"));
|
||||
let declined = false, browserPosts = 0; const saved = new Map<string, any>(); const cleanup: Array<() => Promise<any>> = [];
|
||||
const task = piNativeTasks.find(row => row.id === "human-permission-denial")!;
|
||||
const issue = () => ({ id: "issue", identifier: "PI-1", title: task.buildTitle("fixture"), status: declined ? "done" : "in_progress" });
|
||||
const run = () => ({ id: "run", status: declined ? "succeeded" : "running", runtimeMode: "native", processPid: 123, processGroupId: 123, processStartedAt: "2026-09-29T00:00:00Z" });
|
||||
const request = { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" }, origin: { adapter, provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline", label: "Decline" }] };
|
||||
const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: "pi-human-denied.txt", status: "failed", output: "Pi operation was denied or cancelled" })] : [])];
|
||||
const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: proof.target, status: "failed", output: "Pi operation was denied or cancelled" })] : [])];
|
||||
const api = {
|
||||
post: async () => ({ name: "Pi fixture project" }), patch: async (_path: string, value: any) => value,
|
||||
get: async (path: string) => {
|
||||
@@ -47,11 +52,12 @@ async function exercise(mutation: boolean, remote = false, adapter = "acpx-runti
|
||||
try {
|
||||
if (remote) await writeFile(join(workspacePath, "pi-human-denied.txt"), "POISON HOST COPYBACK");
|
||||
const result = await runPiNativeFlow({ page, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "environment" } }, execution: { task, environment: { id: remote ? "daytona" : "local" }, profile: { qualificationCandidate: "pi" } }, workspacePath, nonce: "fixture", deadlineAt: Date.now() + 2000, restart: async () => {}, observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { saved.set(name, value); }, remoteBootstrap, registerCleanupAssertion: (fn: () => Promise<any>) => cleanup.push(fn) } as any);
|
||||
if (!remote) { expect(proof.target).toMatch(/^pc-denied-[a-zA-Z0-9]+\/pi-human-denied\.txt$/); expect(proof.prompt).toContain(`relative path ${proof.target} and content forbidden`); }
|
||||
expect(browserPosts).toBe(1); expect(result.checks.every(check => check.passed)).toBe(true); expect(saved.has("api-state.json")).toBe(true);
|
||||
expect(cleanup).toHaveLength(1);
|
||||
if (mutation) await expect(cleanup[0]!()).rejects.toThrow("no-effect");
|
||||
if (mutation || incomplete) await expect(cleanup[0]!()).rejects.toThrow("no-effect");
|
||||
else expect((await cleanup[0]!())[0].passed).toBe(true);
|
||||
expect(saved.get(remote ? "pi-remote-1-retirement.json" : "pi-human-denial-retirement.json").passed).toBe(!mutation);
|
||||
expect(saved.get(remote ? "pi-remote-1-retirement.json" : "pi-human-denial-retirement.json").passed).toBe(!mutation && !incomplete);
|
||||
} finally { await rm(workspacePath, { recursive: true, force: true }); }
|
||||
}
|
||||
it("drives actual browser-denial flow and independent retirement proof", async () => exercise(false));
|
||||
@@ -60,3 +66,5 @@ it("rejects an observed create/delete even when final target is absent", async (
|
||||
it("proves browser denial against remote watcher and retirement without trusting a host file", async () => exercise(false, true));
|
||||
|
||||
it.each([false, true])("drives sidecar permission denial with remote=%s", async remote => exercise(false, remote, "acpx-runtime-sidecar"));
|
||||
|
||||
it("rejects incomplete local observation even with zero target mutations", async () => exercise(false, false, "acpx-runtime", true));
|
||||
Reference in new issue
Block a user