mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-20 15:09:57 +02:00
fix(runner-e2e): await durable denial evidence before Stop and sampling
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
72cbc3503d
commit
20154268be
3 files changed
+126
-57
No files matched your search
@@ -236,6 +236,49 @@ export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolea
|
||||
&& ["turn.cancelled", "turn.interrupted"].includes(t.eventType);
|
||||
}
|
||||
|
||||
/** Same failed-edit proof used before Stop and during final settlement. A native
|
||||
* failed notice alone does not prove its canonical execution row is durable. */
|
||||
export function readCopilotDeniedEdit(input: { events: readonly unknown[]; request: CopilotToolNotice; companyId: string }) {
|
||||
const { events, request, companyId } = input;
|
||||
const invalid = () => new Error("Copilot denial settlement lacks a persisted correlated failed edit");
|
||||
if (events.length > 20_000 || !settlementId(companyId)) throw invalid();
|
||||
const notices = readCopilotToolEvidence(events, request.runId);
|
||||
const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]);
|
||||
const requested = notices.filter(n => n.stage === "permission_requested" && same(n));
|
||||
const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n));
|
||||
const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? ""));
|
||||
if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit"
|
||||
|| delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once"
|
||||
|| failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid();
|
||||
const rows = events.map(settlementRecord);
|
||||
const get = (row: Record<string, any>) => {
|
||||
const f = settlementRecord(settlementRecord(row.payload).prpEvent);
|
||||
if (row.runId !== request.runId || row.companyId !== companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0
|
||||
|| f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== request.runId || f.turnId !== request.turnId
|
||||
|| !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0
|
||||
|| f.sourceEventId !== `${f.sourceInstanceId}:${request.runId}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid();
|
||||
return f;
|
||||
};
|
||||
const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); };
|
||||
const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq);
|
||||
const stream = (f: Record<string, any>) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId;
|
||||
if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid();
|
||||
const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType)
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId);
|
||||
if (resolutions.length !== 1) throw invalid();
|
||||
const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload);
|
||||
if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval"
|
||||
|| resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq
|
||||
|| resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid();
|
||||
const toolRows = rows.filter(r => r.eventType === "tool.execution.completed"
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId));
|
||||
if (toolRows.length !== 1) throw invalid();
|
||||
const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload);
|
||||
if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq
|
||||
|| toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid();
|
||||
return { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed };
|
||||
}
|
||||
|
||||
/** Denial closes one permission, not necessarily the provider prompt. Match the
|
||||
* single canonical terminal to the same source stream, then independently bind
|
||||
* the audited controller Stop. Provider emittedAt never orders server actions. */
|
||||
@@ -250,43 +293,11 @@ export function readCopilotDenialSettlement(input: {
|
||||
|| stop.schema !== "paperclip.native-cancellation.v1" || stop.runId !== run.id || stop.companyId !== issue.companyId || stop.issueId !== issue.id
|
||||
|| stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only"
|
||||
|| !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid();
|
||||
const notices = readCopilotToolEvidence(events, request.runId);
|
||||
const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]);
|
||||
const requested = notices.filter(n => n.stage === "permission_requested" && same(n));
|
||||
const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n));
|
||||
const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? ""));
|
||||
if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit"
|
||||
|| delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once"
|
||||
|| failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid();
|
||||
const rows = events.map(settlementRecord);
|
||||
const get = (row: Record<string, any>) => {
|
||||
const f = settlementRecord(settlementRecord(row.payload).prpEvent);
|
||||
if (row.runId !== run.id || row.companyId !== issue.companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0
|
||||
|| f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== run.id || f.turnId !== request.turnId
|
||||
|| !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0
|
||||
|| f.sourceEventId !== `${f.sourceInstanceId}:${run.id}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid();
|
||||
return f;
|
||||
};
|
||||
const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); };
|
||||
const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq);
|
||||
const stream = (f: Record<string, any>) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId;
|
||||
if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid();
|
||||
const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType)
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId);
|
||||
if (resolutions.length !== 1) throw invalid();
|
||||
const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload);
|
||||
if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval"
|
||||
|| resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq
|
||||
|| resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid();
|
||||
const { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed } = readCopilotDeniedEdit({ events, request, companyId: issue.companyId });
|
||||
// Ignore legacy unwrapped log summaries, never use them as provider evidence.
|
||||
const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType)
|
||||
&& settlementRecord(r.payload).prpEvent !== undefined);
|
||||
const toolRows = rows.filter(r => r.eventType === "tool.execution.completed"
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId));
|
||||
if (terminalRows.length !== 1 || toolRows.length !== 1) throw invalid();
|
||||
const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload);
|
||||
if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq
|
||||
|| toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid();
|
||||
if (terminalRows.length !== 1) throw invalid();
|
||||
const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload);
|
||||
if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq
|
||||
|| !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType)
|
||||
|
||||
@@ -7,19 +7,34 @@ import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
|
||||
import { copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js";
|
||||
import { readCopilotDeniedEdit, copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
type Row = Record<string, any>;
|
||||
type Check = { id: string; passed: boolean; detail: string };
|
||||
/** Public run Stop and provider settlement are independent facts. The callback
|
||||
* drains run events only after independent process retirement has been proved. */
|
||||
/** Each persistence barrier reloads durable rows. Stop cannot overtake the
|
||||
* failed edit, and a terminal sample cannot use a pre-terminal event cursor. */
|
||||
export async function settleCopilotDeniedRun(input: {
|
||||
api: Pick<RunnerApi, "post">; request: CopilotToolNotice;
|
||||
awaitRetirement(): Promise<{ events: readonly unknown[]; run: unknown; issue: unknown }>;
|
||||
api: Pick<RunnerApi, "post">; request: CopilotToolNotice; deadlineAt: number;
|
||||
load(): Promise<{ events: readonly unknown[]; run: Row; issue: Row; retired: boolean }>;
|
||||
afterDeniedEdit(): Promise<void>;
|
||||
afterSettlement(): Promise<void>;
|
||||
}) {
|
||||
const poll = (label: string, accept: (state: Awaited<ReturnType<typeof input.load>>) => boolean) => pollUntil({
|
||||
label, deadlineAt: input.deadlineAt, load: input.load, accept, intervalMs: 200,
|
||||
reject: state => ["failed", "timed_out"].includes(state.run.status) ? "Copilot provider run failed" : undefined,
|
||||
});
|
||||
await poll("persisted correlated failed native edit", state => {
|
||||
readCopilotDeniedEdit({ events: state.events, request: input.request, companyId: state.issue.companyId }); return true;
|
||||
});
|
||||
await input.afterDeniedEdit();
|
||||
await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`);
|
||||
return readCopilotDenialSettlement({ ...await input.awaitRetirement(), request: input.request });
|
||||
await poll("correlated provider settlement and retired run", state => {
|
||||
if (!state.retired) return false;
|
||||
readCopilotDenialSettlement({ ...state, request: input.request }); return true;
|
||||
});
|
||||
await input.afterSettlement();
|
||||
return readCopilotDenialSettlement({ ...await input.load(), request: input.request });
|
||||
}
|
||||
|
||||
export async function runCopilotProtectionFlow(input: {
|
||||
@@ -140,14 +155,18 @@ export async function runCopilotProtectionFlow(input: {
|
||||
const clickedAtMs = Date.now(); await card.getByRole("button", { name: "Deny", exact: true }).click();
|
||||
const posted = (await sent).postDataJSON();
|
||||
check("browser-exact-denial", posted.turnId === request.turnId && posted.requestKind === "permission_approval" && posted.resolution?.action === "decline", "Browser submitted denial for the exact run/request/turn");
|
||||
await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed"));
|
||||
await sample("after-decision");
|
||||
const settlement = await settleCopilotDeniedRun({ api, request, awaitRetirement: async () => {
|
||||
await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0)));
|
||||
if (remote) await sealRemote();
|
||||
await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup");
|
||||
return { events: runEvents, run: runs[0], issue };
|
||||
} });
|
||||
const settlement = await settleCopilotDeniedRun({ api, request, deadlineAt: input.deadlineAt,
|
||||
load: async () => {
|
||||
const state = await load();
|
||||
return { events: state.runEvents, run: state.runs[0]!, issue: state.issue,
|
||||
retired: remote || (state.processes.captured && state.processes.live.length === 0) };
|
||||
},
|
||||
afterDeniedEdit: () => sample("after-decision"),
|
||||
afterSettlement: async () => {
|
||||
if (remote) await sealRemote();
|
||||
await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup");
|
||||
},
|
||||
});
|
||||
watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish();
|
||||
const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!;
|
||||
await input.evidence("copilot-denial-settlement.json", settlement);
|
||||
|
||||
@@ -108,14 +108,53 @@ describe("Copilot denial provider settlement and separate audited run Stop", ()
|
||||
f.events[1]!.payload.prpEvent.normalizedSessionId = "foreign";
|
||||
expect(() => copilotDenialSampleCursor(f.events, f.request)).toThrow(/cursor/);
|
||||
});
|
||||
it("flow sends public Stop, awaits retirement, then reads the raced canonical terminal", async () => {
|
||||
const f = fixture(), order: string[] = []; const held = f.events.splice(4, 1)[0];
|
||||
const post = vi.fn(async (url: string) => { order.push(url); f.events.push(held); });
|
||||
const settlement = await settleCopilotDeniedRun({ api: { post } as any, request: f.request,
|
||||
awaitRetirement: async () => { order.push("retired-and-final-events-read"); return f; } });
|
||||
expect(order).toEqual(["/api/heartbeat-runs/run/cancel", "retired-and-final-events-read"]);
|
||||
expect(post).toHaveBeenCalledTimes(1); expect(settlement.branch).toBe("provider_completed_before_stop_settlement");
|
||||
await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request,
|
||||
awaitRetirement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live");
|
||||
it("waits for failed-edit persistence before Stop and terminal persistence before sampling", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const f = fixture("turn.cancelled"), order: string[] = []; let loads = 0;
|
||||
const post = vi.fn(async (url: string) => { order.push(url); });
|
||||
const load = async () => {
|
||||
loads++;
|
||||
// Native failed notice arrives first. Run cancellation and retirement
|
||||
// also become visible before the durable canonical provider terminal.
|
||||
const events = f.events.filter((_, i) => !(loads === 1 && i === 3) && !(loads < 4 && i === 4));
|
||||
order.push(`load-${loads}`);
|
||||
return { ...f, events, run: loads < 3 ? { ...f.run, status: "running" } : f.run, retired: loads >= 3 };
|
||||
};
|
||||
const afterDeniedEdit = vi.fn(async () => { order.push("sample-after-decision"); });
|
||||
const afterSettlement = vi.fn(async () => { order.push("sample-terminal"); });
|
||||
const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 2000,
|
||||
load, afterDeniedEdit, afterSettlement });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(post).not.toHaveBeenCalled(); expect(afterDeniedEdit).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
expect(post).toHaveBeenCalledTimes(1); expect(afterSettlement).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
expect((await result).branch).toBe("provider_cancelled_or_interrupted");
|
||||
expect(order).toEqual(["load-1", "load-2", "sample-after-decision", "/api/heartbeat-runs/run/cancel", "load-3", "load-4", "sample-terminal", "load-5"]);
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
it.each(["missing-edit", "foreign-edit", "missing-terminal", "foreign-terminal", "live-process"])("fails closed on %s without premature side effects or terminal samples", async kind => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {});
|
||||
if (kind === "missing-edit") f.events.splice(3, 1);
|
||||
if (kind === "foreign-edit") f.events[3]!.payload.prpEvent.normalizedSessionId = "foreign";
|
||||
if (kind === "missing-terminal") f.events.splice(4, 1);
|
||||
if (kind === "foreign-terminal") terminal(f).sourceInstanceId = "foreign";
|
||||
const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 500,
|
||||
load: async () => ({ ...f, retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, afterSettlement: sample });
|
||||
const rejected = expect(result).rejects.toThrow(/Timed out waiting/);
|
||||
await vi.advanceTimersByTimeAsync(600); await rejected;
|
||||
expect(post).toHaveBeenCalledTimes(kind.endsWith("edit") ? 0 : 1);
|
||||
expect(sample).not.toHaveBeenCalled();
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
it("retains completed-before-Stop settlement and propagates retirement sampling failure", async () => {
|
||||
const f = fixture(), post = vi.fn(async () => {});
|
||||
const input = { api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000,
|
||||
load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {} };
|
||||
expect((await settleCopilotDeniedRun(input)).branch).toBe("provider_completed_before_stop_settlement");
|
||||
await expect(settleCopilotDeniedRun({ ...input, afterSettlement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user