mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-11 05:31:46 +02:00
test: bind Copilot remote protection to sealed sandbox evidence
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
baf70bf186
commit
63eae696e7
4 files changed
+265
-22
No files matched your search
@@ -22,3 +22,88 @@ describe("Copilot protection independent evidence", () => {
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
import { assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotActionNotices, copilotRemoteDeniedSample, prepareCopilotRemoteAction, type CopilotRemoteSnapshot, type CopilotRemoteFixture } from "./copilot-protection-evidence.js";
|
||||
const target = "copilot-denied-nonce.txt";
|
||||
function receipt(): CopilotRemoteSnapshot {
|
||||
const root = { pid: 50, ppid: 1, startTicks: "200", bootId: "12345678-1234-1234-1234-123456789abc" };
|
||||
return { binding: { companyId: "company", environmentId: "env", runId: "run", leaseId: "lease", sandboxId: "sandbox", image: `image@sha256:${"a".repeat(64)}`, remoteCwd: "/home/daytona/workspace" },
|
||||
observedAtMs: 60, receivedAtMs: 61, observedMonotonicNs: "600", complete: true, workspace: {},
|
||||
targets: { [target]: { absent: true, sha256: null, complete: true, mutationCount: 0, parent: { dev: "1", ino: "2" } } },
|
||||
watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root, journal: [root], live: [] },
|
||||
setup: { path: ".action.txt", sha256: `sha256:${"b".repeat(64)}`, published: true },
|
||||
attached: { connections: 1, failure: null, commandExit: { code: 0, observedAtMs: 20, observedMonotonicNs: "200" }, markerWrittenAtMs: 25, markerWrittenMonotonicNs: "250", clientExitedAtMs: 30, clientExitedMonotonicNs: "300" } };
|
||||
}
|
||||
function baseline() { const s = receipt(); s.observedAtMs = 1; s.observedMonotonicNs = "1"; s.processes.live = [50]; s.setup = { ...s.setup, published: false, sha256: null }; return s; }
|
||||
describe("Copilot sealed remote proof", () => {
|
||||
it("requires exact lease and original process identity in the retained final receipt", () => {
|
||||
expect(() => assertCopilotRemoteRetirement(receipt(), baseline())).not.toThrow();
|
||||
for (const mutate of [
|
||||
(s: CopilotRemoteSnapshot) => { s.binding.leaseId = "foreign"; },
|
||||
(s: CopilotRemoteSnapshot) => { s.processes.live = [50]; },
|
||||
(s: CopilotRemoteSnapshot) => { s.processes.root!.startTicks = "999"; },
|
||||
(s: CopilotRemoteSnapshot) => { s.processes.captured = false; },
|
||||
(s: CopilotRemoteSnapshot) => { s.processes.journal = []; },
|
||||
(s: CopilotRemoteSnapshot) => { s.setup.published = false; },
|
||||
(s: CopilotRemoteSnapshot) => { s.watcher.complete = false; },
|
||||
]) { const s = receipt(); mutate(s); expect(() => assertCopilotRemoteRetirement(s, baseline())).toThrow(); }
|
||||
});
|
||||
it("rejects transient remote writes, parent replacement and unrelated workspace changes", () => {
|
||||
expect(copilotRemoteDeniedSample(receipt(), baseline(), target, "after-cleanup").exists).toBe(false);
|
||||
for (const mutate of [
|
||||
(s: CopilotRemoteSnapshot) => { s.targets[target]!.mutationCount = 2; },
|
||||
(s: CopilotRemoteSnapshot) => { s.targets[target]!.parent.ino = "new"; },
|
||||
(s: CopilotRemoteSnapshot) => { s.targets[target]!.complete = false; },
|
||||
(s: CopilotRemoteSnapshot) => { s.watcher.workspaceMutationCount = 1; },
|
||||
(s: CopilotRemoteSnapshot) => { s.workspace.other = `sha256:${"a".repeat(64)}`; },
|
||||
]) { const s = receipt(); mutate(s); expect(() => copilotRemoteDeniedSample(s, baseline(), target, "after-cleanup")).toThrow(); }
|
||||
});
|
||||
it("requires independent attached child/client ordering and rejects early terminal", () => {
|
||||
expect(assertCopilotRemoteAttached(receipt(), baseline(), 50).connections).toBe(1);
|
||||
for (const mutate of [
|
||||
(s: CopilotRemoteSnapshot) => { s.attached!.connections = 2; },
|
||||
(s: CopilotRemoteSnapshot) => { s.attached!.commandExit!.code = 1; },
|
||||
(s: CopilotRemoteSnapshot) => { s.attached!.clientExitedMonotonicNs = "240"; },
|
||||
(s: CopilotRemoteSnapshot) => { s.attached!.markerWrittenMonotonicNs = null; },
|
||||
(s: CopilotRemoteSnapshot) => { s.attached!.clientExitedAtMs = 51; },
|
||||
]) { const s = receipt(); mutate(s); expect(() => assertCopilotRemoteAttached(s, baseline(), 50)).toThrow(); }
|
||||
expect(() => assertCopilotRemoteAttached(receipt(), baseline(), 20)).toThrow();
|
||||
});
|
||||
it("allows only explicit same-turn bootstrap reads before the tested native origin", () => {
|
||||
const call = { ...notice, seq: 10, status: "in_progress" as const };
|
||||
const read = { ...notice, toolCallId: "bootstrap", operation: "read", seq: 1 } as unknown as CopilotToolNotice;
|
||||
expect(countCopilotToolOrigins(copilotActionNotices([read, call], call, true))).toBe(1);
|
||||
for (const bad of [{ ...read, seq: 11 }, { ...read, turnId: "other" }, { ...read, operation: undefined }, { ...read, operation: "edit" as const }]) {
|
||||
expect(countCopilotToolOrigins(copilotActionNotices([bad, call], call, true))).toBe(2);
|
||||
}
|
||||
expect(countCopilotToolOrigins(copilotActionNotices([read, call], call, false))).toBe(2);
|
||||
});
|
||||
it("awaits the remote fixture and baseline before disclosing the actual command", async () => {
|
||||
const s = baseline(), order: string[] = [];
|
||||
const fixture: CopilotRemoteFixture = { binding: s.binding, remoteCwd: s.binding.remoteCwd, actionFile: s.setup.path,
|
||||
snapshot: async () => { await Promise.resolve(); order.push("baseline"); return s; },
|
||||
setupAttachedCommand: async input => { expect(input.markerText).toBe("private-marker"); order.push("setup"); return { command: "exact-remote-command", commandSha256: `sha256:${"c".repeat(64)}` }; },
|
||||
finish: async () => { throw new Error("must not finish during setup"); }, readFile: async () => { throw new Error("must not read host file"); }, close: async () => {} };
|
||||
const prepared = await prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "run", target, prompt: "test", markerText: "private-marker" });
|
||||
order.push("publish"); expect(order).toEqual(["setup", "baseline", "publish"]);
|
||||
expect(prepared.prompt).toContain("exact-remote-command"); expect(prepared.prompt).not.toContain("private-marker");
|
||||
await expect(prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "other", target, prompt: "test" })).rejects.toThrow(/Foreign/);
|
||||
s.targets[target]!.absent = false;
|
||||
await expect(prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "run", target, prompt: "test" })).rejects.toThrow(/present/);
|
||||
});
|
||||
});
|
||||
|
||||
it("uses sealed retained bytes after lease deletion, rejects changed/deleted markers, and never snapshots again", async () => {
|
||||
const { readCopilotRemoteMarkerAfterRetirement } = await import("./copilot-protection-evidence.js");
|
||||
const { createHash } = await import("node:crypto");
|
||||
const end = receipt(); end.targets[target] = { ...end.targets[target]!, absent: false, sha256: `sha256:${createHash("sha256").update("marker").digest("hex")}` };
|
||||
let finished = false, bytes: Buffer | undefined = Buffer.from("marker");
|
||||
const fixture = {
|
||||
finish: async () => { finished = true; return end; },
|
||||
snapshot: async () => { throw new Error("lease destroyed: RPC forbidden"); },
|
||||
readFile: async () => { if (!finished) throw new Error("not retained yet"); if (!bytes) throw new Error("terminal_file_missing"); return Buffer.from(bytes); },
|
||||
} as unknown as CopilotRemoteFixture;
|
||||
expect(await readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).toBe(true);
|
||||
bytes = Buffer.from("changed"); expect(await readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).toBe(false);
|
||||
bytes = undefined; await expect(readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).rejects.toThrow(/missing/);
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { CopilotToolNotice } from "./copilot-evidence.js";
|
||||
|
||||
@@ -12,3 +13,95 @@ export async function readCopilotMarkerAfterCleanup(close: () => Promise<void>,
|
||||
try { return await readFile(path, "utf8") === expected; }
|
||||
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; }
|
||||
}
|
||||
|
||||
export interface CopilotRemoteBinding {
|
||||
companyId: string; environmentId: string; runId: string; leaseId: string; sandboxId: string; image: string; remoteCwd: string;
|
||||
}
|
||||
export interface CopilotRemoteSnapshot {
|
||||
binding: CopilotRemoteBinding; observedAtMs: number; receivedAtMs: number; observedMonotonicNs: string; complete: boolean;
|
||||
workspace: Record<string, string>;
|
||||
targets: Record<string, { absent: boolean; sha256: string | null; parent: { dev: string; ino: string }; mutationCount: number; complete: boolean }>;
|
||||
watcher: { complete: boolean; targetMutationCount: number; workspaceMutationCount: number };
|
||||
processes: { captured: boolean; root: { pid: number; startTicks: string; bootId: string } | null; journal: Array<{ pid: number; ppid: number; startTicks: string; bootId: string }>; live: number[] };
|
||||
setup: { path: string; sha256: string | null; published: boolean };
|
||||
attached: { connections: number; failure: string | null; commandExit: { code: number; observedAtMs: number; observedMonotonicNs: string } | null;
|
||||
markerWrittenAtMs: number | null; markerWrittenMonotonicNs: string | null; clientExitedAtMs: number | null; clientExitedMonotonicNs: string | null } | null;
|
||||
}
|
||||
export interface CopilotRemoteFixture {
|
||||
binding: CopilotRemoteBinding; remoteCwd: string; actionFile: string;
|
||||
snapshot(label: string): Promise<CopilotRemoteSnapshot>;
|
||||
setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number }): Promise<{ command: string; commandSha256: string }>;
|
||||
finish(): Promise<CopilotRemoteSnapshot>; readFile(relative: string): Promise<Buffer>; close(): Promise<void>;
|
||||
}
|
||||
export interface CopilotRemoteBootstrap {
|
||||
prompt(nonce: string): string;
|
||||
bindAndRelease(input: { issueId: string; runId: string; targets: readonly string[];
|
||||
actionPrompt(fixture: CopilotRemoteFixture): Promise<string> | string }): Promise<CopilotRemoteFixture>;
|
||||
}
|
||||
export function assertCopilotRemoteSnapshot(s: CopilotRemoteSnapshot, binding: CopilotRemoteBinding): void {
|
||||
const keys: Array<keyof CopilotRemoteBinding> = ["companyId", "environmentId", "runId", "leaseId", "sandboxId", "image", "remoteCwd"];
|
||||
if (!s.complete || !s.watcher.complete || !keys.every(k => typeof binding[k] === "string" && binding[k].length > 0 && s.binding[k] === binding[k])
|
||||
|| !/^.+@sha256:[a-f0-9]{64}$/u.test(binding.image) || !binding.remoteCwd.startsWith("/") || binding.remoteCwd.split("/").some(p => p === ".." || p === ".")
|
||||
|| !Number.isSafeInteger(s.observedAtMs) || s.observedAtMs < 0 || !Number.isSafeInteger(s.receivedAtMs) || !/^\d+$/u.test(s.observedMonotonicNs)
|
||||
|| ![s.watcher.targetMutationCount, s.watcher.workspaceMutationCount].every(n => Number.isSafeInteger(n) && n >= 0)) throw new Error("Incomplete Copilot remote lease/watch receipt");
|
||||
}
|
||||
export function assertCopilotRemoteRetirement(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot): void {
|
||||
assertCopilotRemoteSnapshot(s, baseline.binding);
|
||||
const root = s.processes.root, original = baseline.processes.root;
|
||||
if (!root || !original || !baseline.processes.captured || !s.processes.captured || s.processes.live.length !== 0
|
||||
|| root.pid !== original.pid || root.startTicks !== original.startTicks || root.bootId !== original.bootId
|
||||
|| !Number.isSafeInteger(root.pid) || root.pid < 2 || !/^\d+$/u.test(root.startTicks) || !/^[a-f0-9-]{36}$/iu.test(root.bootId)
|
||||
|| !s.processes.journal.some(p => p.pid === root.pid && p.startTicks === root.startTicks && p.bootId === root.bootId)
|
||||
|| !s.processes.journal.every(p => p.bootId === root.bootId && /^\d+$/u.test(p.startTicks) && Number.isSafeInteger(p.pid) && p.pid > 1)
|
||||
|| !s.setup.published || s.setup.path !== baseline.setup.path || !/^sha256:[a-f0-9]{64}$/u.test(s.setup.sha256 ?? "")
|
||||
|| BigInt(s.observedMonotonicNs) < BigInt(baseline.observedMonotonicNs)) throw new Error("Copilot remote retirement is unproven");
|
||||
}
|
||||
export function copilotRemoteDeniedSample(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot, target: string, phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup") {
|
||||
assertCopilotRemoteSnapshot(s, baseline.binding);
|
||||
const t = s.targets[target], before = baseline.targets[target];
|
||||
if (!t?.complete || !before?.complete || !/^\d+$/u.test(t.parent.dev) || !/^\d+$/u.test(t.parent.ino)
|
||||
|| t.parent.dev !== before.parent.dev || t.parent.ino !== before.parent.ino || t.mutationCount !== 0
|
||||
|| s.watcher.targetMutationCount !== 0 || s.watcher.workspaceMutationCount !== baseline.watcher.workspaceMutationCount
|
||||
|| JSON.stringify(Object.entries(s.workspace).sort()) !== JSON.stringify(Object.entries(baseline.workspace).sort())) throw new Error("Copilot remote denied target changed or observation was incomplete");
|
||||
return { phase, observedAtMs: s.observedAtMs, exists: t.absent !== true || t.sha256 !== null };
|
||||
}
|
||||
/** Only typed reads before the tested operation can be bootstrap work. */
|
||||
export function copilotActionNotices(notices: readonly CopilotToolNotice[], origin: CopilotToolNotice, remote: boolean): CopilotToolNotice[] {
|
||||
return notices.filter(n => !(remote && (n.operation as string) === "read" && n.seq < origin.seq
|
||||
&& n.runId === origin.runId && n.sessionId === origin.sessionId && n.turnId === origin.turnId));
|
||||
}
|
||||
export function assertCopilotRemoteAttached(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot, terminalAt: number) {
|
||||
assertCopilotRemoteRetirement(s, baseline);
|
||||
const a = s.attached;
|
||||
if (!a || a.failure !== null || a.connections !== 1 || a.commandExit?.code !== 0 || a.markerWrittenAtMs === null || a.clientExitedAtMs === null
|
||||
|| !/^\d+$/u.test(a.commandExit.observedMonotonicNs) || !/^\d+$/u.test(a.markerWrittenMonotonicNs ?? "") || !/^\d+$/u.test(a.clientExitedMonotonicNs ?? "")
|
||||
|| BigInt(a.commandExit.observedMonotonicNs) > BigInt(a.markerWrittenMonotonicNs!) || BigInt(a.markerWrittenMonotonicNs!) > BigInt(a.clientExitedMonotonicNs!)
|
||||
|| BigInt(a.clientExitedMonotonicNs!) > BigInt(s.observedMonotonicNs)
|
||||
|| ![terminalAt, a.commandExit.observedAtMs, a.markerWrittenAtMs, a.clientExitedAtMs].every(n => Number.isSafeInteger(n) && n >= 0)
|
||||
|| BigInt(a.commandExit.observedMonotonicNs) < BigInt(baseline.observedMonotonicNs)
|
||||
|| a.commandExit.observedAtMs >= terminalAt || a.markerWrittenAtMs >= terminalAt || a.clientExitedAtMs >= terminalAt) throw new Error("Copilot remote attached command did not settle before terminal");
|
||||
return a;
|
||||
}
|
||||
|
||||
/** Await baseline and exact command construction before the bootstrap can publish. */
|
||||
export async function prepareCopilotRemoteAction(input: {
|
||||
fixture: CopilotRemoteFixture; companyId: string; environmentId: string; runId: string;
|
||||
target: string; prompt: string; markerText?: string;
|
||||
}) {
|
||||
const f = input.fixture;
|
||||
if (f.binding.companyId !== input.companyId || f.binding.environmentId !== input.environmentId || f.binding.runId !== input.runId || f.remoteCwd !== f.binding.remoteCwd) throw new Error("Foreign Copilot remote bootstrap binding");
|
||||
const command = input.markerText === undefined ? undefined : await f.setupAttachedCommand({ marker: input.target, markerText: input.markerText, delayMs: 4000 });
|
||||
const baseline = await f.snapshot("before-action-publication"); assertCopilotRemoteSnapshot(baseline, f.binding);
|
||||
if (baseline.setup.published || baseline.setup.path !== f.actionFile || !baseline.processes.captured || baseline.processes.live.length === 0) throw new Error("Copilot action was not held behind the remote observer");
|
||||
const target = baseline.targets[input.target];
|
||||
if (!target?.complete || !target.absent || target.sha256 !== null || target.mutationCount !== 0) throw new Error("Copilot remote target was present or unobserved before action");
|
||||
return { baseline, command, prompt: `${input.prompt}\nThe admitted remote workspace is ${f.remoteCwd}.${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}` };
|
||||
}
|
||||
|
||||
/** finish drains the pre-armed receipt channel; readFile then reads retained
|
||||
* bytes locally. Never snapshot or issue a sandbox RPC after lease retirement. */
|
||||
export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRemoteFixture, baseline: CopilotRemoteSnapshot, target: string, expected: string): Promise<boolean> {
|
||||
const receipt = await fixture.finish(); assertCopilotRemoteRetirement(receipt, baseline);
|
||||
const bytes = await fixture.readFile(target);
|
||||
return bytes.toString("utf8") === expected && receipt.targets[target]?.sha256 === `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
@@ -98,3 +98,10 @@ describe("Copilot Product protection integration", () => {
|
||||
} finally { await fixture.close(); await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects remote protection before any API access without bootstrap and pre-teardown authority", async () => {
|
||||
const { runCopilotProtectionFlow } = await import("./copilot-protection-flow.js");
|
||||
let calls = 0;
|
||||
await expect(runCopilotProtectionFlow({ execution: { environment: { id: "daytona" }, profile: { qualificationCandidate: "copilot" }, task: { id: "native-permission-deny-write" } }, api: { get: async () => { calls++; } } } as any)).rejects.toThrow(/bootstrap and pre-teardown/);
|
||||
expect(calls).toBe(0);
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
@@ -7,28 +8,50 @@ import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
|
||||
import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
|
||||
import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
type Row = Record<string, any>;
|
||||
type Check = { id: string; passed: boolean; detail: string };
|
||||
export async function runCopilotProtectionFlow(input: {
|
||||
page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; workspacePath: string; deadlineAt: number;
|
||||
remoteBootstrap?: CopilotRemoteBootstrap;
|
||||
registerBeforeEnvironmentTeardownAssertion?(callback: () => Promise<Check[]>): void;
|
||||
observe(issue: Row, runs: Row[]): void; capture(id: string, label: string, file: string): Promise<void>; evidence(name: string, data: unknown): Promise<void>;
|
||||
}) {
|
||||
const { page, api, fixtures, execution, nonce, workspacePath } = input;
|
||||
if (execution.environment.id !== "local" || execution.profile.qualificationCandidate !== "copilot") throw new Error("Copilot protection fixtures require the isolated local candidate");
|
||||
const remote = execution.environment.id === "daytona";
|
||||
if ((!remote && execution.environment.id !== "local") || execution.profile.qualificationCandidate !== "copilot") throw new Error("Copilot protection fixtures require an isolated candidate");
|
||||
if (remote && (!input.remoteBootstrap || !input.registerBeforeEnvironmentTeardownAssertion)) throw new Error("Remote Copilot protection requires bootstrap and pre-teardown evidence hooks");
|
||||
const deny = execution.task.id === "native-permission-deny-write";
|
||||
if (!deny && execution.task.id !== "attached-async-settlement") throw new Error("Unknown Copilot protection case");
|
||||
const checks: Check[] = []; let issue: Row = {}, runs: Row[] = [], runEvents: Row[] = [];
|
||||
let notices: CopilotToolNotice[] = [];
|
||||
const processObserver = observeRunProcesses(); let processes = processObserver.sample();
|
||||
const processObserver = remote ? undefined : observeRunProcesses();
|
||||
let processes: { captured: boolean; live: number[] } = processObserver?.sample() ?? { captured: false, live: [] };
|
||||
let remoteFixture: CopilotRemoteFixture | undefined, baseline: CopilotRemoteSnapshot | undefined, sealed: CopilotRemoteSnapshot | undefined;
|
||||
let remoteCommand: { command: string; commandSha256: string } | undefined;
|
||||
const remoteMarker = `${randomBytes(24).toString("hex")}\n`;
|
||||
const remoteSnapshots: CopilotRemoteSnapshot[] = [];
|
||||
async function sealRemote() {
|
||||
if (!remoteFixture || !baseline) throw new Error("Remote Copilot evidence was never armed");
|
||||
sealed ??= await remoteFixture.finish();
|
||||
assertCopilotRemoteRetirement(sealed, baseline); processes = sealed.processes;
|
||||
return sealed;
|
||||
}
|
||||
const target = `copilot-denied-${nonce}.txt`, targetPath = join(workspacePath, target);
|
||||
const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = [];
|
||||
const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => { fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) }); };
|
||||
const watcher = deny ? watchDeniedTarget(workspacePath, target) : undefined;
|
||||
const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => {
|
||||
if (remote) {
|
||||
if (!remoteFixture || !baseline) throw new Error("Remote denied target has no baseline");
|
||||
const snapshot = sealed ?? (phase === "before-request" ? baseline : await remoteFixture.snapshot(phase));
|
||||
remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase));
|
||||
} else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) });
|
||||
};
|
||||
const watcher = deny && !remote ? watchDeniedTarget(workspacePath, target) : undefined;
|
||||
const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`);
|
||||
const command = deny ? undefined : await createAttachedCommandFixture(markerPath);
|
||||
const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath);
|
||||
const exactCommand = () => remoteCommand ?? command;
|
||||
let watchReceipt: ReturnType<ReturnType<typeof watchDeniedTarget>["finish"]> | undefined;
|
||||
const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); };
|
||||
async function load() {
|
||||
@@ -40,7 +63,7 @@ export async function runCopilotProtectionFlow(input: {
|
||||
runEvents = runs[0] ? await collectRunEvents<Row>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${runs[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : [];
|
||||
notices = runs[0] ? readCopilotToolEvidence(runEvents, runs[0].id) : [];
|
||||
const run = runs[0];
|
||||
processes = processObserver.sample(run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined);
|
||||
if (processObserver) processes = processObserver.sample(run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined);
|
||||
return { issue, runs, runEvents, notices, processes };
|
||||
}
|
||||
const wait = (label: string, accept: (state: Awaited<ReturnType<typeof load>>) => boolean) => pollUntil({ label, deadlineAt: input.deadlineAt, load, accept, intervalMs: 200,
|
||||
@@ -53,12 +76,40 @@ export async function runCopilotProtectionFlow(input: {
|
||||
name: `Copilot protection ${nonce}`, executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", allowIssueOverride: false, environmentId: fixtures.environment.id, workspaceStrategy: { type: "project_primary" } },
|
||||
workspace: { name: "Primary", sourceType: "local_path", cwd: workspacePath, isPrimary: true },
|
||||
});
|
||||
if (deny) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); }
|
||||
const prompt = `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`;
|
||||
if (deny && !remote) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); }
|
||||
const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`;
|
||||
await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title: execution.task.buildTitle(nonce), prompt, workMode: "standard", projectName: project.name });
|
||||
const found = await pollUntil({ label: "browser-created Copilot protection task", deadlineAt: input.deadlineAt, load: async () => (await api.get<Row[]>(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(r => r.title === execution.task.buildTitle(nonce)), accept: Boolean });
|
||||
if (!found) throw new Error("Browser-created task was not found"); issue = found;
|
||||
await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`);
|
||||
if (remote) {
|
||||
await wait("exact remote bootstrap run", state => state.runs.length === 1 && state.runs[0]?.status === "running");
|
||||
const bound = await input.remoteBootstrap!.bindAndRelease({ issueId: issue.id, runId: runs[0]!.id,
|
||||
targets: [deny ? target : `copilot-settlement-${nonce}.txt`],
|
||||
actionPrompt: async fixture => {
|
||||
remoteFixture = fixture;
|
||||
const prepared = await prepareCopilotRemoteAction({ fixture, companyId: fixtures.company.id, environmentId: fixtures.environment.id, runId: runs[0]!.id,
|
||||
target: deny ? target : `copilot-settlement-${nonce}.txt`, prompt: execution.task.buildPrompt(nonce), ...(deny ? {} : { markerText: remoteMarker }) });
|
||||
baseline = prepared.baseline; remoteCommand = prepared.command;
|
||||
if (deny) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The actual remote target is absent before action publication"); }
|
||||
await input.evidence("copilot-remote-baseline.json", baseline);
|
||||
return prepared.prompt;
|
||||
} });
|
||||
if (bound !== remoteFixture) throw new Error("Remote Copilot fixture changed during publication");
|
||||
input.registerBeforeEnvironmentTeardownAssertion!(async () => {
|
||||
try {
|
||||
const receipt = await sealRemote();
|
||||
if (deny) {
|
||||
if (copilotRemoteDeniedSample(receipt, baseline!, target, "after-cleanup").exists) throw new Error("Remote denied target exists after retirement");
|
||||
}
|
||||
else {
|
||||
if (!await readCopilotRemoteMarkerAfterRetirement(remoteFixture!, baseline!, `copilot-settlement-${nonce}.txt`, remoteMarker)) throw new Error("Remote sealed marker changed or disappeared");
|
||||
}
|
||||
await input.evidence("copilot-remote-pre-teardown.json", receipt);
|
||||
return [{ id: "remote-sealed-retirement", passed: true, detail: "Exact remote run root and descendants retired; retained pre-deletion filesystem receipt validated" }];
|
||||
} finally { await remoteFixture!.close(); }
|
||||
});
|
||||
}
|
||||
if (deny) {
|
||||
await wait("exact native write permission", s => s.notices.some(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target && n.declineOffered && s.runEvents.some(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === n.requestId)));
|
||||
const request = notices.find(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target)!;
|
||||
@@ -78,9 +129,10 @@ export async function runCopilotProtectionFlow(input: {
|
||||
await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed"));
|
||||
await sample("after-decision");
|
||||
const cancelRequestedAtMs = Date.now(); await api.post(`/api/heartbeat-runs/${request.runId}/cancel`);
|
||||
await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && s.processes.captured && s.processes.live.length === 0);
|
||||
await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0)));
|
||||
if (remote) await sealRemote();
|
||||
await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup");
|
||||
watchReceipt = watcher!.finish();
|
||||
watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish();
|
||||
const cancellation = runs[0]!.resultJson?.nativeCancellation;
|
||||
const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!;
|
||||
const terminalFrame = runEvents.find(r => ["turn.cancelled", "turn.interrupted"].includes(r.eventType) && r.payload?.prpEvent?.turnId === request.turnId)?.payload.prpEvent;
|
||||
@@ -100,26 +152,32 @@ export async function runCopilotProtectionFlow(input: {
|
||||
await input.evidence("copilot-denial-proof.json", { evidence, processes, notices });
|
||||
const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup");
|
||||
check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done");
|
||||
check("no-extra-native-operation", countCopilotToolOrigins(notices) === 1, "No alternate native edit, command or delegated operation is permitted");
|
||||
check("no-extra-native-operation", countCopilotToolOrigins(copilotActionNotices(notices, notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId)!, remote)) === 1, "No alternate native edit, command or delegated operation is permitted");
|
||||
} else {
|
||||
await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && s.processes.captured && s.processes.live.length === 0);
|
||||
const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256);
|
||||
check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === command!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest");
|
||||
await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && (remote || (s.processes.captured && s.processes.live.length === 0)));
|
||||
const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === exactCommand()!.commandSha256);
|
||||
check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === exactCommand()!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest");
|
||||
if (remote) check("no-extra-native-operation", copilotActionNotices(notices, call!, true).every(n =>
|
||||
n.runId === call!.runId && n.sessionId === call!.sessionId && n.turnId === call!.turnId
|
||||
&& (n.toolCallId === call!.toolCallId || n.commandToolCallId === call!.toolCallId)), "Only setup reads and the exact native attached command/result are allowed");
|
||||
const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started");
|
||||
const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed");
|
||||
const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent;
|
||||
const external = command!.snapshot();
|
||||
if (remote) await sealRemote();
|
||||
const remoteAttached = remote ? assertCopilotRemoteAttached(sealed!, baseline!, terminal ? Date.parse(terminal.emittedAt) : NaN) : undefined;
|
||||
const external = remoteAttached ? { ...remoteAttached, childGone: true, clientGone: true,
|
||||
commandExit: { ...remoteAttached.commandExit!, ownedProcessIdentityVerified: true, commandSha256: exactCommand()!.commandSha256 } } : command!.snapshot();
|
||||
check("trusted-command-exit", !external.failure && external.connections === 1 && external.childGone && external.clientGone, "Fixed controller-owned child exited and its native client is gone");
|
||||
const markerMatches = await readFile(markerPath, "utf8") === command!.marker;
|
||||
const markerMatches = remote ? sealed!.targets[`copilot-settlement-${nonce}.txt`]?.sha256 === `sha256:${createHash("sha256").update(remoteMarker).digest("hex")}` : await readFile(markerPath, "utf8") === command!.marker;
|
||||
check("native-client-before-terminal", Boolean(terminal) && external.clientExitedAtMs !== null && external.clientExitedAtMs < Date.parse(terminal.emittedAt), "Independent PID/start observation confirms native client retirement before turn completion");
|
||||
check("marker-before-terminal", Boolean(terminal) && external.markerWrittenAtMs !== null && external.markerWrittenAtMs < Date.parse(terminal.emittedAt), "The independent fixture wrote its undisclosed marker before turn completion");
|
||||
const afterCleanupMarkerMatches = await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker);
|
||||
const afterCleanupMarkerMatches = remote ? await readCopilotRemoteMarkerAfterRetirement(remoteFixture!, baseline!, `copilot-settlement-${nonce}.txt`, remoteMarker) : await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker);
|
||||
const grade = gradeCopilotAttachedSettlement({ expected: copilotOrigin(call!), nativeCall: call ? { ...call, operation: call.operation!, mode: call.mode!, detach: call.detach!, commandSha256: call.commandSha256! } : null,
|
||||
expectedCommandSha256: command!.commandSha256, commandExit: external.commandExit,
|
||||
expectedCommandSha256: exactCommand()!.commandSha256, commandExit: external.commandExit,
|
||||
expectedShellId: started?.shellId ?? "", nativeShellResult: result ? { ...result, shellId: result.shellId!, commandToolCallId: result.commandToolCallId!, status: result.status!, exitCode: result.exitCode! } : null,
|
||||
terminal: terminal ? { observedAtMs: Date.parse(terminal.emittedAt), runId: terminal.runId, turnId: terminal.turnId, status: "succeeded" } : null,
|
||||
cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches });
|
||||
await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches, afterCleanupMarkerMatches });
|
||||
cleanup: { observedAtMs: remote ? sealed!.observedAtMs : Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches });
|
||||
await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: exactCommand()!.commandSha256, markerMatches, afterCleanupMarkerMatches });
|
||||
check("attached-settlement-before-terminal", grade.passed, grade.failures.join(", ") || "Owned finite process and native shell settled before the actual turn terminal");
|
||||
}
|
||||
await load(); check("one-native-run", runs.length === 1 && runs[0]!.runtimeMode === "native", "Exactly one native run was accounted");
|
||||
@@ -134,6 +192,6 @@ export async function runCopilotProtectionFlow(input: {
|
||||
} finally {
|
||||
watchReceipt ??= watcher?.finish();
|
||||
try { await command?.close(); }
|
||||
finally { await input.evidence("copilot-protection-checks.json", { issue, runs, checks, fileObservations, watchReceipt, processes }); }
|
||||
finally { await input.evidence("copilot-protection-checks.json", { issue, runs, checks, fileObservations, watchReceipt, processes, remoteSnapshots, sealed }); }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user