mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-14 08:26:46 +02:00
test(runner-e2e): honor provider permission evidence order
Generate both input orders through the real Cursor and Copilot projectors. Preserve Cursor's deferred permission evidence invariant while allowing Copilot's immediate permission notice before its exact tool origin; retain all pre-Stop and settlement bindings. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
90a4d86417
commit
35fbc123a7
4 files changed
+77
-9
No files matched your search
@@ -422,9 +422,12 @@ exist in both pre-Stop API observations. The v2 pending receipt adds the native
|
||||
origin and tool-start row hashes and source sequences. The fresh reread and
|
||||
settlement must preserve those exact rows; a later tool start cannot backfill
|
||||
missing pre-Stop evidence. Command/path, request, tool, turn, session and source
|
||||
checks remain strict. The native evidence projector still requires the exact
|
||||
tool origin before it can emit correlated permission evidence; this does not
|
||||
claim the original ACP wire order. Old v1 receipts are not valid inputs to the
|
||||
checks remain strict. Cursor's native evidence projector still requires the
|
||||
exact tool origin before it can emit correlated permission evidence. Copilot
|
||||
emits permission evidence immediately, so its permission notice may also precede
|
||||
the native tool notice. Calibration tests exercise both actual projectors in
|
||||
both input orders. Neither policy claims the original ACP wire order of a live
|
||||
attempt. Old v1 receipts are not valid inputs to the
|
||||
new grader. Earlier paid failures retain their original definition and grade.
|
||||
|
||||
`paperclip.e2e.native-active-stop-settlement.v1` accepts only
|
||||
|
||||
@@ -1479,6 +1479,9 @@ sequence through settlement. Missing, changed or replayed evidence cannot be
|
||||
filled in after Stop. This is a new grader definition; old v1 pending receipts
|
||||
and failed paid attempts are not regraded. No original provider wire order is
|
||||
inferred from API insertion or display order.
|
||||
Native notice ordering follows each real projector: Cursor waits for its exact
|
||||
command origin before emitting permission evidence; Copilot can emit the
|
||||
permission notice first. Both still require the exact tool origin before Stop.
|
||||
|
||||
Passage requires a canonical cancelled request and cancelled provider turn,
|
||||
exact caller-intent acknowledgement, an unfinished task, rejection of a later
|
||||
|
||||
@@ -93,7 +93,10 @@ function origin(events: readonly unknown[], scope: ActiveStopScope, bootstrap?:
|
||||
const tools = selected.filter(n => n.stage === "tool");
|
||||
fail(tools.filter(n => n.status === "pending").length === 1
|
||||
&& tools.filter(n => n.status === "failed").length <= 1
|
||||
&& tools[0]!.status === "pending" && tools[0]!.seq < notice.seq
|
||||
&& tools[0]!.status === "pending"
|
||||
// Cursor queues permission evidence until its exact command origin exists.
|
||||
// Copilot projects the request immediately, including permission-first ACP.
|
||||
&& (scope.provider !== "cursor" || tools[0]!.seq < notice.seq)
|
||||
&& selected.filter(n => n.stage === "permission_delivered").length <= 1, "duplicate native operation lifecycle");
|
||||
const executions = rows.filter(x => x.event.eventType.startsWith("tool.execution.") && rec(x.event.payload).executionId === bootstrapReadExecutionId(notice.toolCallId));
|
||||
fail(executions.filter(x => x.event.eventType === "tool.execution.started").length === 1
|
||||
|
||||
@@ -4,6 +4,9 @@ import { assertActiveStopRetirement, readActiveStopRemoteRetirement, type Active
|
||||
import { stopAtPendingPermission } from "./native-active-stop-flow.js";
|
||||
import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js";
|
||||
import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js";
|
||||
import { createCursorToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/cursor-tool-evidence.js";
|
||||
import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js";
|
||||
import type { CanonicalProviderEvent } from "../../packages/paperclip-runner/src/provider-events.js";
|
||||
|
||||
const caller = readActiveStopCaller({ deploymentMode: "local_trusted" }, { session: { userId: "local-board", id: "paperclip:local_implicit:local-board" } });
|
||||
const cancellationRequestId = "11111111-2222-4333-8444-555555555555";
|
||||
@@ -47,8 +50,8 @@ const arrivalOrders = ["tool-first", "permission-first"] as const;
|
||||
function withArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrders[number]) {
|
||||
const f = fixture(provider);
|
||||
if (order === "permission-first") {
|
||||
// The card can be published while the provider callback is held. Native
|
||||
// permission evidence still requires the later exact tool origin to flush.
|
||||
// Isolate canonical card/start order in these synthetic mutation cases.
|
||||
// Actual provider-specific notice ordering is exercised by the projectors.
|
||||
f.events.unshift(f.events.pop()!);
|
||||
f.events.forEach((row, index) => {
|
||||
row.seq = frame(row).sourceSeq = index + 1;
|
||||
@@ -57,6 +60,36 @@ function withArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrd
|
||||
}
|
||||
return f;
|
||||
}
|
||||
function withProjectedArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrders[number]) {
|
||||
const f = fixture(provider), start = payload(f.events[1]!), card = payload(f.events[3]!);
|
||||
const command = "printf MUST_NOT_EXIST > target.txt";
|
||||
if (provider === "cursor") {
|
||||
f.scope.commandSha256 = `sha256:${createHash("sha256").update(command).digest("hex")}`;
|
||||
start.commandSha256 = f.scope.commandSha256;
|
||||
}
|
||||
f.events.length = 0;
|
||||
const append = (eventType: string, value: Row) => f.events.push(f.row(f.events.length + 1, eventType, value));
|
||||
const projector = (provider === "cursor" ? createCursorToolEvidence : createCopilotToolEvidence)({
|
||||
sessionId: "native-session", turnId: "turn", workingDirectory: "/fixture", active: () => true,
|
||||
emit: (event: CanonicalProviderEvent) => { append(event.eventType, event.payload); },
|
||||
});
|
||||
const tool = { type: "tool_call", tag: "tool_call", toolCallId: "tool", status: "pending",
|
||||
...(provider === "cursor" ? { kind: "execute", rawInput: { command } } : { kind: "edit", rawInput: { fileName: "target.txt" } }),
|
||||
};
|
||||
let delivered: ((outcome: string) => void) | undefined, beforeTool: Row[] | undefined;
|
||||
const permission = () => {
|
||||
delivered = projector.permission({ raw: { sessionId: "native-session", toolCall: provider === "cursor"
|
||||
? { toolCallId: "tool", kind: "execute" } : { toolCallId: "tool", kind: "edit", rawInput: tool.rawInput } } }, "request", ["decline", "cancel"]);
|
||||
// Match the sidecar: invoke the evidence projector before publishing the
|
||||
// permission request; project each iterator tool before its canonical row.
|
||||
append("runtime_request.created", card);
|
||||
};
|
||||
const notification = () => { projector.tool(tool); append("tool.execution.started", start); };
|
||||
if (order === "permission-first") { permission(); beforeTool = structuredClone(f.events); notification(); }
|
||||
else { notification(); permission(); }
|
||||
if (!delivered) throw new Error("Native permission projector rejected the fixture");
|
||||
return { ...f, beforeTool, delivered, nativeStageOrder: () => f.events.filter(row => row.eventType === "provider.notice.recorded").map(row => payload(row).provenance.eventType) };
|
||||
}
|
||||
function settled() { const f = fixture(); const pending = f.pending(); f.settle(); return { f, pending, read: () => readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() }) }; }
|
||||
function withSessionPrefix(provider: ActiveStopProvider = "cursor") {
|
||||
const f = fixture(provider);
|
||||
@@ -81,7 +114,9 @@ function withSessionPrefix(provider: ActiveStopProvider = "cursor") {
|
||||
describe("definitely active native permission Stop", () => {
|
||||
describe.each(["cursor", "copilot"] as const)("%s pre-Stop arrival orders", provider => {
|
||||
it.each(arrivalOrders)("accepts %s only after both exact origins exist and retains them through settlement", order => {
|
||||
const f = withArrivalOrder(provider, order), pending = f.pending();
|
||||
const f = withProjectedArrivalOrder(provider, order), pending = f.pending();
|
||||
expect(f.nativeStageOrder()).toEqual(provider === "copilot" && order === "permission-first"
|
||||
? ["permission_requested", "tool"] : ["tool", "permission_requested"]);
|
||||
expect(pending.schema).toBe("paperclip.e2e.native-active-stop-pending.v2");
|
||||
expect(pending.toolOriginRowSha256).toMatch(/^sha256:[a-f0-9]{64}$/u);
|
||||
expect(pending.toolStartedRowSha256).toMatch(/^sha256:[a-f0-9]{64}$/u);
|
||||
@@ -90,6 +125,19 @@ describe("definitely active native permission Stop", () => {
|
||||
expect(readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() }))
|
||||
.toMatchObject({ branch: "pending_permission_cancelled", pending });
|
||||
});
|
||||
it("cannot retain permission-only projector output before its exact origin arrives", () => {
|
||||
const f = withProjectedArrivalOrder(provider, "permission-first");
|
||||
expect(() => observeActiveStopPending({ ...f.state(), events: f.beforeTool!, scope: f.scope, caller, cancellationRequestId })).toThrow();
|
||||
});
|
||||
it.each(arrivalOrders)("rejects a delivered answer in real-projector %s evidence", order => {
|
||||
const f = withProjectedArrivalOrder(provider, order); f.delivered("cancel"); expect(f.pending).toThrow("permission already answered");
|
||||
});
|
||||
it("does not accept a permission-first request for a different native origin", () => {
|
||||
const f = withProjectedArrivalOrder(provider, "permission-first");
|
||||
const tool = f.events.find(row => row.eventType === "provider.notice.recorded" && payload(row).provenance.eventType === "tool")!;
|
||||
payload(tool).details.find((d: Row) => d.name === "toolCallId").value = "another-tool";
|
||||
expect(f.pending).toThrow("native operation origin missing");
|
||||
});
|
||||
it.each([
|
||||
["missing native origin", (f: ReturnType<typeof fixture>) => { f.events.splice(1, 1); }],
|
||||
["missing canonical start", (f: ReturnType<typeof fixture>) => { f.events.splice(2, 1); }],
|
||||
@@ -120,6 +168,14 @@ describe("definitely active native permission Stop", () => {
|
||||
.toThrow("observed pending request changed");
|
||||
});
|
||||
});
|
||||
it("still rejects a Cursor permission notice before the origin its projector requires", () => {
|
||||
const f = withProjectedArrivalOrder("cursor", "permission-first");
|
||||
[f.events[1], f.events[2]] = [f.events[2]!, f.events[1]!];
|
||||
f.events.forEach((row, index) => {
|
||||
row.seq = frame(row).sourceSeq = index + 1; frame(row).sourceEventId = `source:run:${index + 1}`;
|
||||
});
|
||||
expect(f.pending).toThrow("duplicate native operation lifecycle");
|
||||
});
|
||||
it.each(["cursor", "copilot"] as const)("accepts the mixed v1/v2 session prefix before strict %s pending proof", provider => {
|
||||
const f = withSessionPrefix(provider);
|
||||
expect(f.pending()).toMatchObject({ requestId: "request", toolCallId: "tool", permissionSourceSeq: 6, requestSourceSeq: 7 });
|
||||
@@ -240,8 +296,11 @@ describe("definitely active native permission Stop", () => {
|
||||
});
|
||||
|
||||
describe("active Stop flow wiring", () => {
|
||||
it.each(arrivalOrders)("awaits retention and rechecks %s evidence before issuing the single caller UUID request", async arrivalOrder => {
|
||||
const f = withArrivalOrder("cursor", arrivalOrder); const order: string[] = []; let retainedId = "";
|
||||
it.each([
|
||||
["cursor", "tool-first"], ["cursor", "permission-first"],
|
||||
["copilot", "tool-first"], ["copilot", "permission-first"],
|
||||
] as const)("awaits retention and rechecks real %s %s evidence before issuing the single caller UUID request", async (provider, arrivalOrder) => {
|
||||
const f = withProjectedArrivalOrder(provider, arrivalOrder); const order: string[] = []; let retainedId = "";
|
||||
const stop = vi.fn(async (runId: string, id: string) => { expect(runId).toBe("run"); expect(id).toBe(retainedId); order.push("stop"); return f.settle(id); });
|
||||
const result = await stopAtPendingPermission({ scope: f.scope, caller, deadlineAt: Date.now() + 1000,
|
||||
load: async () => { order.push("load"); return f.state(); },
|
||||
|
||||
Reference in new issue
Block a user