From 20154268be8d89f1d28cec91beb5fc95c1b675be Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 10:57:39 -0500 Subject: [PATCH] fix(runner-e2e): await durable denial evidence before Stop and sampling Co-Authored-By: Paperclip --- .../runner-e2e/copilot-protection-evidence.ts | 79 +++++++++++-------- tests/runner-e2e/copilot-protection-flow.ts | 47 +++++++---- .../copilot-protection-settlement.test.ts | 57 ++++++++++--- 3 files changed, 126 insertions(+), 57 deletions(-) diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 4e9a6a39c88..65dc9c64016 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -236,6 +236,49 @@ export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolea && ["turn.cancelled", "turn.interrupted"].includes(t.eventType); } +/** Same failed-edit proof used before Stop and during final settlement. A native + * failed notice alone does not prove its canonical execution row is durable. */ +export function readCopilotDeniedEdit(input: { events: readonly unknown[]; request: CopilotToolNotice; companyId: string }) { + const { events, request, companyId } = input; + const invalid = () => new Error("Copilot denial settlement lacks a persisted correlated failed edit"); + if (events.length > 20_000 || !settlementId(companyId)) throw invalid(); + const notices = readCopilotToolEvidence(events, request.runId); + const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]); + const requested = notices.filter(n => n.stage === "permission_requested" && same(n)); + const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n)); + const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? "")); + if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit" + || delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once" + || failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid(); + const rows = events.map(settlementRecord); + const get = (row: Record) => { + const f = settlementRecord(settlementRecord(row.payload).prpEvent); + if (row.runId !== request.runId || row.companyId !== companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0 + || f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== request.runId || f.turnId !== request.turnId + || !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0 + || f.sourceEventId !== `${f.sourceInstanceId}:${request.runId}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid(); + return f; + }; + const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); }; + const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq); + const stream = (f: Record) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId; + if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid(); + const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType) + && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId); + if (resolutions.length !== 1) throw invalid(); + const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload); + if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval" + || resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq + || resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid(); + const toolRows = rows.filter(r => r.eventType === "tool.execution.completed" + && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId)); + if (toolRows.length !== 1) throw invalid(); + const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload); + if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq + || toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid(); + return { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed }; +} + /** Denial closes one permission, not necessarily the provider prompt. Match the * single canonical terminal to the same source stream, then independently bind * the audited controller Stop. Provider emittedAt never orders server actions. */ @@ -250,43 +293,11 @@ export function readCopilotDenialSettlement(input: { || stop.schema !== "paperclip.native-cancellation.v1" || stop.runId !== run.id || stop.companyId !== issue.companyId || stop.issueId !== issue.id || stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only" || !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid(); - const notices = readCopilotToolEvidence(events, request.runId); - const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]); - const requested = notices.filter(n => n.stage === "permission_requested" && same(n)); - const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n)); - const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? "")); - if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit" - || delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once" - || failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid(); - const rows = events.map(settlementRecord); - const get = (row: Record) => { - const f = settlementRecord(settlementRecord(row.payload).prpEvent); - if (row.runId !== run.id || row.companyId !== issue.companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0 - || f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== run.id || f.turnId !== request.turnId - || !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0 - || f.sourceEventId !== `${f.sourceInstanceId}:${run.id}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid(); - return f; - }; - const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); }; - const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq); - const stream = (f: Record) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId; - if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid(); - const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType) - && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId); - if (resolutions.length !== 1) throw invalid(); - const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload); - if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval" - || resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq - || resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid(); + const { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed } = readCopilotDeniedEdit({ events, request, companyId: issue.companyId }); // Ignore legacy unwrapped log summaries, never use them as provider evidence. const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType) && settlementRecord(r.payload).prpEvent !== undefined); - const toolRows = rows.filter(r => r.eventType === "tool.execution.completed" - && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId)); - if (terminalRows.length !== 1 || toolRows.length !== 1) throw invalid(); - const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload); - if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq - || toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid(); + if (terminalRows.length !== 1) throw invalid(); const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload); if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq || !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType) diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 5bd70634f03..904821a3f31 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,19 +7,34 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; -import { copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; +import { readCopilotDeniedEdit, copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; type Check = { id: string; passed: boolean; detail: string }; -/** Public run Stop and provider settlement are independent facts. The callback - * drains run events only after independent process retirement has been proved. */ +/** Each persistence barrier reloads durable rows. Stop cannot overtake the + * failed edit, and a terminal sample cannot use a pre-terminal event cursor. */ export async function settleCopilotDeniedRun(input: { - api: Pick; request: CopilotToolNotice; - awaitRetirement(): Promise<{ events: readonly unknown[]; run: unknown; issue: unknown }>; + api: Pick; request: CopilotToolNotice; deadlineAt: number; + load(): Promise<{ events: readonly unknown[]; run: Row; issue: Row; retired: boolean }>; + afterDeniedEdit(): Promise; + afterSettlement(): Promise; }) { + const poll = (label: string, accept: (state: Awaited>) => boolean) => pollUntil({ + label, deadlineAt: input.deadlineAt, load: input.load, accept, intervalMs: 200, + reject: state => ["failed", "timed_out"].includes(state.run.status) ? "Copilot provider run failed" : undefined, + }); + await poll("persisted correlated failed native edit", state => { + readCopilotDeniedEdit({ events: state.events, request: input.request, companyId: state.issue.companyId }); return true; + }); + await input.afterDeniedEdit(); await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`); - return readCopilotDenialSettlement({ ...await input.awaitRetirement(), request: input.request }); + await poll("correlated provider settlement and retired run", state => { + if (!state.retired) return false; + readCopilotDenialSettlement({ ...state, request: input.request }); return true; + }); + await input.afterSettlement(); + return readCopilotDenialSettlement({ ...await input.load(), request: input.request }); } export async function runCopilotProtectionFlow(input: { @@ -140,14 +155,18 @@ export async function runCopilotProtectionFlow(input: { const clickedAtMs = Date.now(); await card.getByRole("button", { name: "Deny", exact: true }).click(); const posted = (await sent).postDataJSON(); check("browser-exact-denial", posted.turnId === request.turnId && posted.requestKind === "permission_approval" && posted.resolution?.action === "decline", "Browser submitted denial for the exact run/request/turn"); - await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")); - await sample("after-decision"); - const settlement = await settleCopilotDeniedRun({ api, request, awaitRetirement: async () => { - await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0))); - if (remote) await sealRemote(); - await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); - return { events: runEvents, run: runs[0], issue }; - } }); + const settlement = await settleCopilotDeniedRun({ api, request, deadlineAt: input.deadlineAt, + load: async () => { + const state = await load(); + return { events: state.runEvents, run: state.runs[0]!, issue: state.issue, + retired: remote || (state.processes.captured && state.processes.live.length === 0) }; + }, + afterDeniedEdit: () => sample("after-decision"), + afterSettlement: async () => { + if (remote) await sealRemote(); + await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); + }, + }); watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish(); const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!; await input.evidence("copilot-denial-settlement.json", settlement); diff --git a/tests/runner-e2e/copilot-protection-settlement.test.ts b/tests/runner-e2e/copilot-protection-settlement.test.ts index a25cb49f57a..276e371e937 100644 --- a/tests/runner-e2e/copilot-protection-settlement.test.ts +++ b/tests/runner-e2e/copilot-protection-settlement.test.ts @@ -108,14 +108,53 @@ describe("Copilot denial provider settlement and separate audited run Stop", () f.events[1]!.payload.prpEvent.normalizedSessionId = "foreign"; expect(() => copilotDenialSampleCursor(f.events, f.request)).toThrow(/cursor/); }); - it("flow sends public Stop, awaits retirement, then reads the raced canonical terminal", async () => { - const f = fixture(), order: string[] = []; const held = f.events.splice(4, 1)[0]; - const post = vi.fn(async (url: string) => { order.push(url); f.events.push(held); }); - const settlement = await settleCopilotDeniedRun({ api: { post } as any, request: f.request, - awaitRetirement: async () => { order.push("retired-and-final-events-read"); return f; } }); - expect(order).toEqual(["/api/heartbeat-runs/run/cancel", "retired-and-final-events-read"]); - expect(post).toHaveBeenCalledTimes(1); expect(settlement.branch).toBe("provider_completed_before_stop_settlement"); - await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, - awaitRetirement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live"); + it("waits for failed-edit persistence before Stop and terminal persistence before sampling", async () => { + vi.useFakeTimers(); + try { + const f = fixture("turn.cancelled"), order: string[] = []; let loads = 0; + const post = vi.fn(async (url: string) => { order.push(url); }); + const load = async () => { + loads++; + // Native failed notice arrives first. Run cancellation and retirement + // also become visible before the durable canonical provider terminal. + const events = f.events.filter((_, i) => !(loads === 1 && i === 3) && !(loads < 4 && i === 4)); + order.push(`load-${loads}`); + return { ...f, events, run: loads < 3 ? { ...f.run, status: "running" } : f.run, retired: loads >= 3 }; + }; + const afterDeniedEdit = vi.fn(async () => { order.push("sample-after-decision"); }); + const afterSettlement = vi.fn(async () => { order.push("sample-terminal"); }); + const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 2000, + load, afterDeniedEdit, afterSettlement }); + await vi.advanceTimersByTimeAsync(0); + expect(post).not.toHaveBeenCalled(); expect(afterDeniedEdit).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(200); + expect(post).toHaveBeenCalledTimes(1); expect(afterSettlement).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(200); + expect((await result).branch).toBe("provider_cancelled_or_interrupted"); + expect(order).toEqual(["load-1", "load-2", "sample-after-decision", "/api/heartbeat-runs/run/cancel", "load-3", "load-4", "sample-terminal", "load-5"]); + } finally { vi.useRealTimers(); } + }); + it.each(["missing-edit", "foreign-edit", "missing-terminal", "foreign-terminal", "live-process"])("fails closed on %s without premature side effects or terminal samples", async kind => { + vi.useFakeTimers(); + try { + const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {}); + if (kind === "missing-edit") f.events.splice(3, 1); + if (kind === "foreign-edit") f.events[3]!.payload.prpEvent.normalizedSessionId = "foreign"; + if (kind === "missing-terminal") f.events.splice(4, 1); + if (kind === "foreign-terminal") terminal(f).sourceInstanceId = "foreign"; + const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 500, + load: async () => ({ ...f, retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, afterSettlement: sample }); + const rejected = expect(result).rejects.toThrow(/Timed out waiting/); + await vi.advanceTimersByTimeAsync(600); await rejected; + expect(post).toHaveBeenCalledTimes(kind.endsWith("edit") ? 0 : 1); + expect(sample).not.toHaveBeenCalled(); + } finally { vi.useRealTimers(); } + }); + it("retains completed-before-Stop settlement and propagates retirement sampling failure", async () => { + const f = fixture(), post = vi.fn(async () => {}); + const input = { api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, + load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {} }; + expect((await settleCopilotDeniedRun(input)).branch).toBe("provider_completed_before_stop_settlement"); + await expect(settleCopilotDeniedRun({ ...input, afterSettlement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live"); }); });