diff --git a/tests/runner-e2e/copilot-protection-evidence.test.ts b/tests/runner-e2e/copilot-protection-evidence.test.ts index 70bef038c0..f314c5659d 100644 --- a/tests/runner-e2e/copilot-protection-evidence.test.ts +++ b/tests/runner-e2e/copilot-protection-evidence.test.ts @@ -22,3 +22,88 @@ describe("Copilot protection independent evidence", () => { } finally { await rm(root, { recursive: true, force: true }); } }); }); + +import { assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotActionNotices, copilotRemoteDeniedSample, prepareCopilotRemoteAction, type CopilotRemoteSnapshot, type CopilotRemoteFixture } from "./copilot-protection-evidence.js"; +const target = "copilot-denied-nonce.txt"; +function receipt(): CopilotRemoteSnapshot { + const root = { pid: 50, ppid: 1, startTicks: "200", bootId: "12345678-1234-1234-1234-123456789abc" }; + return { binding: { companyId: "company", environmentId: "env", runId: "run", leaseId: "lease", sandboxId: "sandbox", image: `image@sha256:${"a".repeat(64)}`, remoteCwd: "/home/daytona/workspace" }, + observedAtMs: 60, receivedAtMs: 61, observedMonotonicNs: "600", complete: true, workspace: {}, + targets: { [target]: { absent: true, sha256: null, complete: true, mutationCount: 0, parent: { dev: "1", ino: "2" } } }, + watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root, journal: [root], live: [] }, + setup: { path: ".action.txt", sha256: `sha256:${"b".repeat(64)}`, published: true }, + attached: { connections: 1, failure: null, commandExit: { code: 0, observedAtMs: 20, observedMonotonicNs: "200" }, markerWrittenAtMs: 25, markerWrittenMonotonicNs: "250", clientExitedAtMs: 30, clientExitedMonotonicNs: "300" } }; +} +function baseline() { const s = receipt(); s.observedAtMs = 1; s.observedMonotonicNs = "1"; s.processes.live = [50]; s.setup = { ...s.setup, published: false, sha256: null }; return s; } +describe("Copilot sealed remote proof", () => { + it("requires exact lease and original process identity in the retained final receipt", () => { + expect(() => assertCopilotRemoteRetirement(receipt(), baseline())).not.toThrow(); + for (const mutate of [ + (s: CopilotRemoteSnapshot) => { s.binding.leaseId = "foreign"; }, + (s: CopilotRemoteSnapshot) => { s.processes.live = [50]; }, + (s: CopilotRemoteSnapshot) => { s.processes.root!.startTicks = "999"; }, + (s: CopilotRemoteSnapshot) => { s.processes.captured = false; }, + (s: CopilotRemoteSnapshot) => { s.processes.journal = []; }, + (s: CopilotRemoteSnapshot) => { s.setup.published = false; }, + (s: CopilotRemoteSnapshot) => { s.watcher.complete = false; }, + ]) { const s = receipt(); mutate(s); expect(() => assertCopilotRemoteRetirement(s, baseline())).toThrow(); } + }); + it("rejects transient remote writes, parent replacement and unrelated workspace changes", () => { + expect(copilotRemoteDeniedSample(receipt(), baseline(), target, "after-cleanup").exists).toBe(false); + for (const mutate of [ + (s: CopilotRemoteSnapshot) => { s.targets[target]!.mutationCount = 2; }, + (s: CopilotRemoteSnapshot) => { s.targets[target]!.parent.ino = "new"; }, + (s: CopilotRemoteSnapshot) => { s.targets[target]!.complete = false; }, + (s: CopilotRemoteSnapshot) => { s.watcher.workspaceMutationCount = 1; }, + (s: CopilotRemoteSnapshot) => { s.workspace.other = `sha256:${"a".repeat(64)}`; }, + ]) { const s = receipt(); mutate(s); expect(() => copilotRemoteDeniedSample(s, baseline(), target, "after-cleanup")).toThrow(); } + }); + it("requires independent attached child/client ordering and rejects early terminal", () => { + expect(assertCopilotRemoteAttached(receipt(), baseline(), 50).connections).toBe(1); + for (const mutate of [ + (s: CopilotRemoteSnapshot) => { s.attached!.connections = 2; }, + (s: CopilotRemoteSnapshot) => { s.attached!.commandExit!.code = 1; }, + (s: CopilotRemoteSnapshot) => { s.attached!.clientExitedMonotonicNs = "240"; }, + (s: CopilotRemoteSnapshot) => { s.attached!.markerWrittenMonotonicNs = null; }, + (s: CopilotRemoteSnapshot) => { s.attached!.clientExitedAtMs = 51; }, + ]) { const s = receipt(); mutate(s); expect(() => assertCopilotRemoteAttached(s, baseline(), 50)).toThrow(); } + expect(() => assertCopilotRemoteAttached(receipt(), baseline(), 20)).toThrow(); + }); + it("allows only explicit same-turn bootstrap reads before the tested native origin", () => { + const call = { ...notice, seq: 10, status: "in_progress" as const }; + const read = { ...notice, toolCallId: "bootstrap", operation: "read", seq: 1 } as unknown as CopilotToolNotice; + expect(countCopilotToolOrigins(copilotActionNotices([read, call], call, true))).toBe(1); + for (const bad of [{ ...read, seq: 11 }, { ...read, turnId: "other" }, { ...read, operation: undefined }, { ...read, operation: "edit" as const }]) { + expect(countCopilotToolOrigins(copilotActionNotices([bad, call], call, true))).toBe(2); + } + expect(countCopilotToolOrigins(copilotActionNotices([read, call], call, false))).toBe(2); + }); + it("awaits the remote fixture and baseline before disclosing the actual command", async () => { + const s = baseline(), order: string[] = []; + const fixture: CopilotRemoteFixture = { binding: s.binding, remoteCwd: s.binding.remoteCwd, actionFile: s.setup.path, + snapshot: async () => { await Promise.resolve(); order.push("baseline"); return s; }, + setupAttachedCommand: async input => { expect(input.markerText).toBe("private-marker"); order.push("setup"); return { command: "exact-remote-command", commandSha256: `sha256:${"c".repeat(64)}` }; }, + finish: async () => { throw new Error("must not finish during setup"); }, readFile: async () => { throw new Error("must not read host file"); }, close: async () => {} }; + const prepared = await prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "run", target, prompt: "test", markerText: "private-marker" }); + order.push("publish"); expect(order).toEqual(["setup", "baseline", "publish"]); + expect(prepared.prompt).toContain("exact-remote-command"); expect(prepared.prompt).not.toContain("private-marker"); + await expect(prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "other", target, prompt: "test" })).rejects.toThrow(/Foreign/); + s.targets[target]!.absent = false; + await expect(prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "run", target, prompt: "test" })).rejects.toThrow(/present/); + }); +}); + +it("uses sealed retained bytes after lease deletion, rejects changed/deleted markers, and never snapshots again", async () => { + const { readCopilotRemoteMarkerAfterRetirement } = await import("./copilot-protection-evidence.js"); + const { createHash } = await import("node:crypto"); + const end = receipt(); end.targets[target] = { ...end.targets[target]!, absent: false, sha256: `sha256:${createHash("sha256").update("marker").digest("hex")}` }; + let finished = false, bytes: Buffer | undefined = Buffer.from("marker"); + const fixture = { + finish: async () => { finished = true; return end; }, + snapshot: async () => { throw new Error("lease destroyed: RPC forbidden"); }, + readFile: async () => { if (!finished) throw new Error("not retained yet"); if (!bytes) throw new Error("terminal_file_missing"); return Buffer.from(bytes); }, + } as unknown as CopilotRemoteFixture; + expect(await readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).toBe(true); + bytes = Buffer.from("changed"); expect(await readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).toBe(false); + bytes = undefined; await expect(readCopilotRemoteMarkerAfterRetirement(fixture, baseline(), target, "marker")).rejects.toThrow(/missing/); +}); diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 01759b0e40..3f19690dab 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import type { CopilotToolNotice } from "./copilot-evidence.js"; @@ -12,3 +13,95 @@ export async function readCopilotMarkerAfterCleanup(close: () => Promise, try { return await readFile(path, "utf8") === expected; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } } + +export interface CopilotRemoteBinding { + companyId: string; environmentId: string; runId: string; leaseId: string; sandboxId: string; image: string; remoteCwd: string; +} +export interface CopilotRemoteSnapshot { + binding: CopilotRemoteBinding; observedAtMs: number; receivedAtMs: number; observedMonotonicNs: string; complete: boolean; + workspace: Record; + targets: Record; + watcher: { complete: boolean; targetMutationCount: number; workspaceMutationCount: number }; + processes: { captured: boolean; root: { pid: number; startTicks: string; bootId: string } | null; journal: Array<{ pid: number; ppid: number; startTicks: string; bootId: string }>; live: number[] }; + setup: { path: string; sha256: string | null; published: boolean }; + attached: { connections: number; failure: string | null; commandExit: { code: number; observedAtMs: number; observedMonotonicNs: string } | null; + markerWrittenAtMs: number | null; markerWrittenMonotonicNs: string | null; clientExitedAtMs: number | null; clientExitedMonotonicNs: string | null } | null; +} +export interface CopilotRemoteFixture { + binding: CopilotRemoteBinding; remoteCwd: string; actionFile: string; + snapshot(label: string): Promise; + setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number }): Promise<{ command: string; commandSha256: string }>; + finish(): Promise; readFile(relative: string): Promise; close(): Promise; +} +export interface CopilotRemoteBootstrap { + prompt(nonce: string): string; + bindAndRelease(input: { issueId: string; runId: string; targets: readonly string[]; + actionPrompt(fixture: CopilotRemoteFixture): Promise | string }): Promise; +} +export function assertCopilotRemoteSnapshot(s: CopilotRemoteSnapshot, binding: CopilotRemoteBinding): void { + const keys: Array = ["companyId", "environmentId", "runId", "leaseId", "sandboxId", "image", "remoteCwd"]; + if (!s.complete || !s.watcher.complete || !keys.every(k => typeof binding[k] === "string" && binding[k].length > 0 && s.binding[k] === binding[k]) + || !/^.+@sha256:[a-f0-9]{64}$/u.test(binding.image) || !binding.remoteCwd.startsWith("/") || binding.remoteCwd.split("/").some(p => p === ".." || p === ".") + || !Number.isSafeInteger(s.observedAtMs) || s.observedAtMs < 0 || !Number.isSafeInteger(s.receivedAtMs) || !/^\d+$/u.test(s.observedMonotonicNs) + || ![s.watcher.targetMutationCount, s.watcher.workspaceMutationCount].every(n => Number.isSafeInteger(n) && n >= 0)) throw new Error("Incomplete Copilot remote lease/watch receipt"); +} +export function assertCopilotRemoteRetirement(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot): void { + assertCopilotRemoteSnapshot(s, baseline.binding); + const root = s.processes.root, original = baseline.processes.root; + if (!root || !original || !baseline.processes.captured || !s.processes.captured || s.processes.live.length !== 0 + || root.pid !== original.pid || root.startTicks !== original.startTicks || root.bootId !== original.bootId + || !Number.isSafeInteger(root.pid) || root.pid < 2 || !/^\d+$/u.test(root.startTicks) || !/^[a-f0-9-]{36}$/iu.test(root.bootId) + || !s.processes.journal.some(p => p.pid === root.pid && p.startTicks === root.startTicks && p.bootId === root.bootId) + || !s.processes.journal.every(p => p.bootId === root.bootId && /^\d+$/u.test(p.startTicks) && Number.isSafeInteger(p.pid) && p.pid > 1) + || !s.setup.published || s.setup.path !== baseline.setup.path || !/^sha256:[a-f0-9]{64}$/u.test(s.setup.sha256 ?? "") + || BigInt(s.observedMonotonicNs) < BigInt(baseline.observedMonotonicNs)) throw new Error("Copilot remote retirement is unproven"); +} +export function copilotRemoteDeniedSample(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot, target: string, phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup") { + assertCopilotRemoteSnapshot(s, baseline.binding); + const t = s.targets[target], before = baseline.targets[target]; + if (!t?.complete || !before?.complete || !/^\d+$/u.test(t.parent.dev) || !/^\d+$/u.test(t.parent.ino) + || t.parent.dev !== before.parent.dev || t.parent.ino !== before.parent.ino || t.mutationCount !== 0 + || s.watcher.targetMutationCount !== 0 || s.watcher.workspaceMutationCount !== baseline.watcher.workspaceMutationCount + || JSON.stringify(Object.entries(s.workspace).sort()) !== JSON.stringify(Object.entries(baseline.workspace).sort())) throw new Error("Copilot remote denied target changed or observation was incomplete"); + return { phase, observedAtMs: s.observedAtMs, exists: t.absent !== true || t.sha256 !== null }; +} +/** Only typed reads before the tested operation can be bootstrap work. */ +export function copilotActionNotices(notices: readonly CopilotToolNotice[], origin: CopilotToolNotice, remote: boolean): CopilotToolNotice[] { + return notices.filter(n => !(remote && (n.operation as string) === "read" && n.seq < origin.seq + && n.runId === origin.runId && n.sessionId === origin.sessionId && n.turnId === origin.turnId)); +} +export function assertCopilotRemoteAttached(s: CopilotRemoteSnapshot, baseline: CopilotRemoteSnapshot, terminalAt: number) { + assertCopilotRemoteRetirement(s, baseline); + const a = s.attached; + if (!a || a.failure !== null || a.connections !== 1 || a.commandExit?.code !== 0 || a.markerWrittenAtMs === null || a.clientExitedAtMs === null + || !/^\d+$/u.test(a.commandExit.observedMonotonicNs) || !/^\d+$/u.test(a.markerWrittenMonotonicNs ?? "") || !/^\d+$/u.test(a.clientExitedMonotonicNs ?? "") + || BigInt(a.commandExit.observedMonotonicNs) > BigInt(a.markerWrittenMonotonicNs!) || BigInt(a.markerWrittenMonotonicNs!) > BigInt(a.clientExitedMonotonicNs!) + || BigInt(a.clientExitedMonotonicNs!) > BigInt(s.observedMonotonicNs) + || ![terminalAt, a.commandExit.observedAtMs, a.markerWrittenAtMs, a.clientExitedAtMs].every(n => Number.isSafeInteger(n) && n >= 0) + || BigInt(a.commandExit.observedMonotonicNs) < BigInt(baseline.observedMonotonicNs) + || a.commandExit.observedAtMs >= terminalAt || a.markerWrittenAtMs >= terminalAt || a.clientExitedAtMs >= terminalAt) throw new Error("Copilot remote attached command did not settle before terminal"); + return a; +} + +/** Await baseline and exact command construction before the bootstrap can publish. */ +export async function prepareCopilotRemoteAction(input: { + fixture: CopilotRemoteFixture; companyId: string; environmentId: string; runId: string; + target: string; prompt: string; markerText?: string; +}) { + const f = input.fixture; + if (f.binding.companyId !== input.companyId || f.binding.environmentId !== input.environmentId || f.binding.runId !== input.runId || f.remoteCwd !== f.binding.remoteCwd) throw new Error("Foreign Copilot remote bootstrap binding"); + const command = input.markerText === undefined ? undefined : await f.setupAttachedCommand({ marker: input.target, markerText: input.markerText, delayMs: 4000 }); + const baseline = await f.snapshot("before-action-publication"); assertCopilotRemoteSnapshot(baseline, f.binding); + if (baseline.setup.published || baseline.setup.path !== f.actionFile || !baseline.processes.captured || baseline.processes.live.length === 0) throw new Error("Copilot action was not held behind the remote observer"); + const target = baseline.targets[input.target]; + if (!target?.complete || !target.absent || target.sha256 !== null || target.mutationCount !== 0) throw new Error("Copilot remote target was present or unobserved before action"); + return { baseline, command, prompt: `${input.prompt}\nThe admitted remote workspace is ${f.remoteCwd}.${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}` }; +} + +/** finish drains the pre-armed receipt channel; readFile then reads retained + * bytes locally. Never snapshot or issue a sandbox RPC after lease retirement. */ +export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRemoteFixture, baseline: CopilotRemoteSnapshot, target: string, expected: string): Promise { + const receipt = await fixture.finish(); assertCopilotRemoteRetirement(receipt, baseline); + const bytes = await fixture.readFile(target); + return bytes.toString("utf8") === expected && receipt.targets[target]?.sha256 === `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 2ca4193cfc..c36b7b5e52 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -98,3 +98,10 @@ describe("Copilot Product protection integration", () => { } finally { await fixture.close(); await rm(root, { recursive: true, force: true }); } }); }); + +it("rejects remote protection before any API access without bootstrap and pre-teardown authority", async () => { + const { runCopilotProtectionFlow } = await import("./copilot-protection-flow.js"); + let calls = 0; + await expect(runCopilotProtectionFlow({ execution: { environment: { id: "daytona" }, profile: { qualificationCandidate: "copilot" }, task: { id: "native-permission-deny-write" } }, api: { get: async () => { calls++; } } } as any)).rejects.toThrow(/bootstrap and pre-teardown/); + expect(calls).toBe(0); +}); diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index f3f0bac023..3238f379db 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -1,3 +1,4 @@ +import { createHash, randomBytes } from "node:crypto"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, type Page } from "@playwright/test"; @@ -7,28 +8,50 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; -import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; +import { readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; type Check = { id: string; passed: boolean; detail: string }; export async function runCopilotProtectionFlow(input: { page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; workspacePath: string; deadlineAt: number; + remoteBootstrap?: CopilotRemoteBootstrap; + registerBeforeEnvironmentTeardownAssertion?(callback: () => Promise): void; observe(issue: Row, runs: Row[]): void; capture(id: string, label: string, file: string): Promise; evidence(name: string, data: unknown): Promise; }) { const { page, api, fixtures, execution, nonce, workspacePath } = input; - if (execution.environment.id !== "local" || execution.profile.qualificationCandidate !== "copilot") throw new Error("Copilot protection fixtures require the isolated local candidate"); + const remote = execution.environment.id === "daytona"; + if ((!remote && execution.environment.id !== "local") || execution.profile.qualificationCandidate !== "copilot") throw new Error("Copilot protection fixtures require an isolated candidate"); + if (remote && (!input.remoteBootstrap || !input.registerBeforeEnvironmentTeardownAssertion)) throw new Error("Remote Copilot protection requires bootstrap and pre-teardown evidence hooks"); const deny = execution.task.id === "native-permission-deny-write"; if (!deny && execution.task.id !== "attached-async-settlement") throw new Error("Unknown Copilot protection case"); const checks: Check[] = []; let issue: Row = {}, runs: Row[] = [], runEvents: Row[] = []; let notices: CopilotToolNotice[] = []; - const processObserver = observeRunProcesses(); let processes = processObserver.sample(); + const processObserver = remote ? undefined : observeRunProcesses(); + let processes: { captured: boolean; live: number[] } = processObserver?.sample() ?? { captured: false, live: [] }; + let remoteFixture: CopilotRemoteFixture | undefined, baseline: CopilotRemoteSnapshot | undefined, sealed: CopilotRemoteSnapshot | undefined; + let remoteCommand: { command: string; commandSha256: string } | undefined; + const remoteMarker = `${randomBytes(24).toString("hex")}\n`; + const remoteSnapshots: CopilotRemoteSnapshot[] = []; + async function sealRemote() { + if (!remoteFixture || !baseline) throw new Error("Remote Copilot evidence was never armed"); + sealed ??= await remoteFixture.finish(); + assertCopilotRemoteRetirement(sealed, baseline); processes = sealed.processes; + return sealed; + } const target = `copilot-denied-${nonce}.txt`, targetPath = join(workspacePath, target); const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = []; - const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => { fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) }); }; - const watcher = deny ? watchDeniedTarget(workspacePath, target) : undefined; + const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => { + if (remote) { + if (!remoteFixture || !baseline) throw new Error("Remote denied target has no baseline"); + const snapshot = sealed ?? (phase === "before-request" ? baseline : await remoteFixture.snapshot(phase)); + remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase)); + } else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) }); + }; + const watcher = deny && !remote ? watchDeniedTarget(workspacePath, target) : undefined; const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`); - const command = deny ? undefined : await createAttachedCommandFixture(markerPath); + const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath); + const exactCommand = () => remoteCommand ?? command; let watchReceipt: ReturnType["finish"]> | undefined; const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); }; async function load() { @@ -40,7 +63,7 @@ export async function runCopilotProtectionFlow(input: { runEvents = runs[0] ? await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${runs[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : []; notices = runs[0] ? readCopilotToolEvidence(runEvents, runs[0].id) : []; const run = runs[0]; - processes = processObserver.sample(run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined); + if (processObserver) processes = processObserver.sample(run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined); return { issue, runs, runEvents, notices, processes }; } const wait = (label: string, accept: (state: Awaited>) => boolean) => pollUntil({ label, deadlineAt: input.deadlineAt, load, accept, intervalMs: 200, @@ -53,12 +76,40 @@ export async function runCopilotProtectionFlow(input: { name: `Copilot protection ${nonce}`, executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", allowIssueOverride: false, environmentId: fixtures.environment.id, workspaceStrategy: { type: "project_primary" } }, workspace: { name: "Primary", sourceType: "local_path", cwd: workspacePath, isPrimary: true }, }); - if (deny) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); } - const prompt = `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`; + if (deny && !remote) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The exact isolated target is absent before dispatch"); } + const prompt = remote ? input.remoteBootstrap!.prompt(nonce) : `${execution.task.buildPrompt(nonce)}${command ? `\nThe exact supplied command is:\n${command.command}\nDo not inspect or modify fixture code, fabricate its marker, or launch a substitute command.` : ""}`; await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title: execution.task.buildTitle(nonce), prompt, workMode: "standard", projectName: project.name }); const found = await pollUntil({ label: "browser-created Copilot protection task", deadlineAt: input.deadlineAt, load: async () => (await api.get(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(r => r.title === execution.task.buildTitle(nonce)), accept: Boolean }); if (!found) throw new Error("Browser-created task was not found"); issue = found; await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`); + if (remote) { + await wait("exact remote bootstrap run", state => state.runs.length === 1 && state.runs[0]?.status === "running"); + const bound = await input.remoteBootstrap!.bindAndRelease({ issueId: issue.id, runId: runs[0]!.id, + targets: [deny ? target : `copilot-settlement-${nonce}.txt`], + actionPrompt: async fixture => { + remoteFixture = fixture; + const prepared = await prepareCopilotRemoteAction({ fixture, companyId: fixtures.company.id, environmentId: fixtures.environment.id, runId: runs[0]!.id, + target: deny ? target : `copilot-settlement-${nonce}.txt`, prompt: execution.task.buildPrompt(nonce), ...(deny ? {} : { markerText: remoteMarker }) }); + baseline = prepared.baseline; remoteCommand = prepared.command; + if (deny) { await sample("before-request"); check("target-initially-absent", !fileObservations[0]!.exists, "The actual remote target is absent before action publication"); } + await input.evidence("copilot-remote-baseline.json", baseline); + return prepared.prompt; + } }); + if (bound !== remoteFixture) throw new Error("Remote Copilot fixture changed during publication"); + input.registerBeforeEnvironmentTeardownAssertion!(async () => { + try { + const receipt = await sealRemote(); + if (deny) { + if (copilotRemoteDeniedSample(receipt, baseline!, target, "after-cleanup").exists) throw new Error("Remote denied target exists after retirement"); + } + else { + if (!await readCopilotRemoteMarkerAfterRetirement(remoteFixture!, baseline!, `copilot-settlement-${nonce}.txt`, remoteMarker)) throw new Error("Remote sealed marker changed or disappeared"); + } + await input.evidence("copilot-remote-pre-teardown.json", receipt); + return [{ id: "remote-sealed-retirement", passed: true, detail: "Exact remote run root and descendants retired; retained pre-deletion filesystem receipt validated" }]; + } finally { await remoteFixture!.close(); } + }); + } if (deny) { await wait("exact native write permission", s => s.notices.some(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target && n.declineOffered && s.runEvents.some(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === n.requestId))); const request = notices.find(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target)!; @@ -78,9 +129,10 @@ export async function runCopilotProtectionFlow(input: { await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")); await sample("after-decision"); const cancelRequestedAtMs = Date.now(); await api.post(`/api/heartbeat-runs/${request.runId}/cancel`); - await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && s.processes.captured && s.processes.live.length === 0); + await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0))); + if (remote) await sealRemote(); await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); - watchReceipt = watcher!.finish(); + watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish(); const cancellation = runs[0]!.resultJson?.nativeCancellation; const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!; const terminalFrame = runEvents.find(r => ["turn.cancelled", "turn.interrupted"].includes(r.eventType) && r.payload?.prpEvent?.turnId === request.turnId)?.payload.prpEvent; @@ -100,26 +152,32 @@ export async function runCopilotProtectionFlow(input: { await input.evidence("copilot-denial-proof.json", { evidence, processes, notices }); const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup"); check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done"); - check("no-extra-native-operation", countCopilotToolOrigins(notices) === 1, "No alternate native edit, command or delegated operation is permitted"); + check("no-extra-native-operation", countCopilotToolOrigins(copilotActionNotices(notices, notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId)!, remote)) === 1, "No alternate native edit, command or delegated operation is permitted"); } else { - await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && s.processes.captured && s.processes.live.length === 0); - const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256); - check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === command!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest"); + await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && (remote || (s.processes.captured && s.processes.live.length === 0))); + const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === exactCommand()!.commandSha256); + check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === exactCommand()!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest"); + if (remote) check("no-extra-native-operation", copilotActionNotices(notices, call!, true).every(n => + n.runId === call!.runId && n.sessionId === call!.sessionId && n.turnId === call!.turnId + && (n.toolCallId === call!.toolCallId || n.commandToolCallId === call!.toolCallId)), "Only setup reads and the exact native attached command/result are allowed"); const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started"); const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed"); const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent; - const external = command!.snapshot(); + if (remote) await sealRemote(); + const remoteAttached = remote ? assertCopilotRemoteAttached(sealed!, baseline!, terminal ? Date.parse(terminal.emittedAt) : NaN) : undefined; + const external = remoteAttached ? { ...remoteAttached, childGone: true, clientGone: true, + commandExit: { ...remoteAttached.commandExit!, ownedProcessIdentityVerified: true, commandSha256: exactCommand()!.commandSha256 } } : command!.snapshot(); check("trusted-command-exit", !external.failure && external.connections === 1 && external.childGone && external.clientGone, "Fixed controller-owned child exited and its native client is gone"); - const markerMatches = await readFile(markerPath, "utf8") === command!.marker; + const markerMatches = remote ? sealed!.targets[`copilot-settlement-${nonce}.txt`]?.sha256 === `sha256:${createHash("sha256").update(remoteMarker).digest("hex")}` : await readFile(markerPath, "utf8") === command!.marker; check("native-client-before-terminal", Boolean(terminal) && external.clientExitedAtMs !== null && external.clientExitedAtMs < Date.parse(terminal.emittedAt), "Independent PID/start observation confirms native client retirement before turn completion"); check("marker-before-terminal", Boolean(terminal) && external.markerWrittenAtMs !== null && external.markerWrittenAtMs < Date.parse(terminal.emittedAt), "The independent fixture wrote its undisclosed marker before turn completion"); - const afterCleanupMarkerMatches = await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker); + const afterCleanupMarkerMatches = remote ? await readCopilotRemoteMarkerAfterRetirement(remoteFixture!, baseline!, `copilot-settlement-${nonce}.txt`, remoteMarker) : await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker); const grade = gradeCopilotAttachedSettlement({ expected: copilotOrigin(call!), nativeCall: call ? { ...call, operation: call.operation!, mode: call.mode!, detach: call.detach!, commandSha256: call.commandSha256! } : null, - expectedCommandSha256: command!.commandSha256, commandExit: external.commandExit, + expectedCommandSha256: exactCommand()!.commandSha256, commandExit: external.commandExit, expectedShellId: started?.shellId ?? "", nativeShellResult: result ? { ...result, shellId: result.shellId!, commandToolCallId: result.commandToolCallId!, status: result.status!, exitCode: result.exitCode! } : null, terminal: terminal ? { observedAtMs: Date.parse(terminal.emittedAt), runId: terminal.runId, turnId: terminal.turnId, status: "succeeded" } : null, - cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches }); - await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches, afterCleanupMarkerMatches }); + cleanup: { observedAtMs: remote ? sealed!.observedAtMs : Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches }); + await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: exactCommand()!.commandSha256, markerMatches, afterCleanupMarkerMatches }); check("attached-settlement-before-terminal", grade.passed, grade.failures.join(", ") || "Owned finite process and native shell settled before the actual turn terminal"); } await load(); check("one-native-run", runs.length === 1 && runs[0]!.runtimeMode === "native", "Exactly one native run was accounted"); @@ -134,6 +192,6 @@ export async function runCopilotProtectionFlow(input: { } finally { watchReceipt ??= watcher?.finish(); try { await command?.close(); } - finally { await input.evidence("copilot-protection-checks.json", { issue, runs, checks, fileObservations, watchReceipt, processes }); } + finally { await input.evidence("copilot-protection-checks.json", { issue, runs, checks, fileObservations, watchReceipt, processes, remoteSnapshots, sealed }); } } }