diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 6a996b9bcf..139c3bd479 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -422,9 +422,12 @@ exist in both pre-Stop API observations. The v2 pending receipt adds the native origin and tool-start row hashes and source sequences. The fresh reread and settlement must preserve those exact rows; a later tool start cannot backfill missing pre-Stop evidence. Command/path, request, tool, turn, session and source -checks remain strict. The native evidence projector still requires the exact -tool origin before it can emit correlated permission evidence; this does not -claim the original ACP wire order. Old v1 receipts are not valid inputs to the +checks remain strict. Cursor's native evidence projector still requires the +exact tool origin before it can emit correlated permission evidence. Copilot +emits permission evidence immediately, so its permission notice may also precede +the native tool notice. Calibration tests exercise both actual projectors in +both input orders. Neither policy claims the original ACP wire order of a live +attempt. Old v1 receipts are not valid inputs to the new grader. Earlier paid failures retain their original definition and grade. `paperclip.e2e.native-active-stop-settlement.v1` accepts only diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c8d20a50ed..ee1d7c0ac0 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1479,6 +1479,9 @@ sequence through settlement. Missing, changed or replayed evidence cannot be filled in after Stop. This is a new grader definition; old v1 pending receipts and failed paid attempts are not regraded. No original provider wire order is inferred from API insertion or display order. +Native notice ordering follows each real projector: Cursor waits for its exact +command origin before emitting permission evidence; Copilot can emit the +permission notice first. Both still require the exact tool origin before Stop. Passage requires a canonical cancelled request and cancelled provider turn, exact caller-intent acknowledgement, an unfinished task, rejection of a later diff --git a/tests/runner-e2e/native-active-stop-evidence.ts b/tests/runner-e2e/native-active-stop-evidence.ts index c8f31226f9..566baf4290 100644 --- a/tests/runner-e2e/native-active-stop-evidence.ts +++ b/tests/runner-e2e/native-active-stop-evidence.ts @@ -93,7 +93,10 @@ function origin(events: readonly unknown[], scope: ActiveStopScope, bootstrap?: const tools = selected.filter(n => n.stage === "tool"); fail(tools.filter(n => n.status === "pending").length === 1 && tools.filter(n => n.status === "failed").length <= 1 - && tools[0]!.status === "pending" && tools[0]!.seq < notice.seq + && tools[0]!.status === "pending" + // Cursor queues permission evidence until its exact command origin exists. + // Copilot projects the request immediately, including permission-first ACP. + && (scope.provider !== "cursor" || tools[0]!.seq < notice.seq) && selected.filter(n => n.stage === "permission_delivered").length <= 1, "duplicate native operation lifecycle"); const executions = rows.filter(x => x.event.eventType.startsWith("tool.execution.") && rec(x.event.payload).executionId === bootstrapReadExecutionId(notice.toolCallId)); fail(executions.filter(x => x.event.eventType === "tool.execution.started").length === 1 diff --git a/tests/runner-e2e/native-active-stop.test.ts b/tests/runner-e2e/native-active-stop.test.ts index a6e4ef6ad1..f6317468b6 100644 --- a/tests/runner-e2e/native-active-stop.test.ts +++ b/tests/runner-e2e/native-active-stop.test.ts @@ -4,6 +4,9 @@ import { assertActiveStopRetirement, readActiveStopRemoteRetirement, type Active import { stopAtPendingPermission } from "./native-active-stop-flow.js"; import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; +import { createCursorToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/cursor-tool-evidence.js"; +import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js"; +import type { CanonicalProviderEvent } from "../../packages/paperclip-runner/src/provider-events.js"; const caller = readActiveStopCaller({ deploymentMode: "local_trusted" }, { session: { userId: "local-board", id: "paperclip:local_implicit:local-board" } }); const cancellationRequestId = "11111111-2222-4333-8444-555555555555"; @@ -47,8 +50,8 @@ const arrivalOrders = ["tool-first", "permission-first"] as const; function withArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrders[number]) { const f = fixture(provider); if (order === "permission-first") { - // The card can be published while the provider callback is held. Native - // permission evidence still requires the later exact tool origin to flush. + // Isolate canonical card/start order in these synthetic mutation cases. + // Actual provider-specific notice ordering is exercised by the projectors. f.events.unshift(f.events.pop()!); f.events.forEach((row, index) => { row.seq = frame(row).sourceSeq = index + 1; @@ -57,6 +60,36 @@ function withArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrd } return f; } +function withProjectedArrivalOrder(provider: ActiveStopProvider, order: typeof arrivalOrders[number]) { + const f = fixture(provider), start = payload(f.events[1]!), card = payload(f.events[3]!); + const command = "printf MUST_NOT_EXIST > target.txt"; + if (provider === "cursor") { + f.scope.commandSha256 = `sha256:${createHash("sha256").update(command).digest("hex")}`; + start.commandSha256 = f.scope.commandSha256; + } + f.events.length = 0; + const append = (eventType: string, value: Row) => f.events.push(f.row(f.events.length + 1, eventType, value)); + const projector = (provider === "cursor" ? createCursorToolEvidence : createCopilotToolEvidence)({ + sessionId: "native-session", turnId: "turn", workingDirectory: "/fixture", active: () => true, + emit: (event: CanonicalProviderEvent) => { append(event.eventType, event.payload); }, + }); + const tool = { type: "tool_call", tag: "tool_call", toolCallId: "tool", status: "pending", + ...(provider === "cursor" ? { kind: "execute", rawInput: { command } } : { kind: "edit", rawInput: { fileName: "target.txt" } }), + }; + let delivered: ((outcome: string) => void) | undefined, beforeTool: Row[] | undefined; + const permission = () => { + delivered = projector.permission({ raw: { sessionId: "native-session", toolCall: provider === "cursor" + ? { toolCallId: "tool", kind: "execute" } : { toolCallId: "tool", kind: "edit", rawInput: tool.rawInput } } }, "request", ["decline", "cancel"]); + // Match the sidecar: invoke the evidence projector before publishing the + // permission request; project each iterator tool before its canonical row. + append("runtime_request.created", card); + }; + const notification = () => { projector.tool(tool); append("tool.execution.started", start); }; + if (order === "permission-first") { permission(); beforeTool = structuredClone(f.events); notification(); } + else { notification(); permission(); } + if (!delivered) throw new Error("Native permission projector rejected the fixture"); + return { ...f, beforeTool, delivered, nativeStageOrder: () => f.events.filter(row => row.eventType === "provider.notice.recorded").map(row => payload(row).provenance.eventType) }; +} function settled() { const f = fixture(); const pending = f.pending(); f.settle(); return { f, pending, read: () => readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() }) }; } function withSessionPrefix(provider: ActiveStopProvider = "cursor") { const f = fixture(provider); @@ -81,7 +114,9 @@ function withSessionPrefix(provider: ActiveStopProvider = "cursor") { describe("definitely active native permission Stop", () => { describe.each(["cursor", "copilot"] as const)("%s pre-Stop arrival orders", provider => { it.each(arrivalOrders)("accepts %s only after both exact origins exist and retains them through settlement", order => { - const f = withArrivalOrder(provider, order), pending = f.pending(); + const f = withProjectedArrivalOrder(provider, order), pending = f.pending(); + expect(f.nativeStageOrder()).toEqual(provider === "copilot" && order === "permission-first" + ? ["permission_requested", "tool"] : ["tool", "permission_requested"]); expect(pending.schema).toBe("paperclip.e2e.native-active-stop-pending.v2"); expect(pending.toolOriginRowSha256).toMatch(/^sha256:[a-f0-9]{64}$/u); expect(pending.toolStartedRowSha256).toMatch(/^sha256:[a-f0-9]{64}$/u); @@ -90,6 +125,19 @@ describe("definitely active native permission Stop", () => { expect(readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() })) .toMatchObject({ branch: "pending_permission_cancelled", pending }); }); + it("cannot retain permission-only projector output before its exact origin arrives", () => { + const f = withProjectedArrivalOrder(provider, "permission-first"); + expect(() => observeActiveStopPending({ ...f.state(), events: f.beforeTool!, scope: f.scope, caller, cancellationRequestId })).toThrow(); + }); + it.each(arrivalOrders)("rejects a delivered answer in real-projector %s evidence", order => { + const f = withProjectedArrivalOrder(provider, order); f.delivered("cancel"); expect(f.pending).toThrow("permission already answered"); + }); + it("does not accept a permission-first request for a different native origin", () => { + const f = withProjectedArrivalOrder(provider, "permission-first"); + const tool = f.events.find(row => row.eventType === "provider.notice.recorded" && payload(row).provenance.eventType === "tool")!; + payload(tool).details.find((d: Row) => d.name === "toolCallId").value = "another-tool"; + expect(f.pending).toThrow("native operation origin missing"); + }); it.each([ ["missing native origin", (f: ReturnType) => { f.events.splice(1, 1); }], ["missing canonical start", (f: ReturnType) => { f.events.splice(2, 1); }], @@ -120,6 +168,14 @@ describe("definitely active native permission Stop", () => { .toThrow("observed pending request changed"); }); }); + it("still rejects a Cursor permission notice before the origin its projector requires", () => { + const f = withProjectedArrivalOrder("cursor", "permission-first"); + [f.events[1], f.events[2]] = [f.events[2]!, f.events[1]!]; + f.events.forEach((row, index) => { + row.seq = frame(row).sourceSeq = index + 1; frame(row).sourceEventId = `source:run:${index + 1}`; + }); + expect(f.pending).toThrow("duplicate native operation lifecycle"); + }); it.each(["cursor", "copilot"] as const)("accepts the mixed v1/v2 session prefix before strict %s pending proof", provider => { const f = withSessionPrefix(provider); expect(f.pending()).toMatchObject({ requestId: "request", toolCallId: "tool", permissionSourceSeq: 6, requestSourceSeq: 7 }); @@ -240,8 +296,11 @@ describe("definitely active native permission Stop", () => { }); describe("active Stop flow wiring", () => { - it.each(arrivalOrders)("awaits retention and rechecks %s evidence before issuing the single caller UUID request", async arrivalOrder => { - const f = withArrivalOrder("cursor", arrivalOrder); const order: string[] = []; let retainedId = ""; + it.each([ + ["cursor", "tool-first"], ["cursor", "permission-first"], + ["copilot", "tool-first"], ["copilot", "permission-first"], + ] as const)("awaits retention and rechecks real %s %s evidence before issuing the single caller UUID request", async (provider, arrivalOrder) => { + const f = withProjectedArrivalOrder(provider, arrivalOrder); const order: string[] = []; let retainedId = ""; const stop = vi.fn(async (runId: string, id: string) => { expect(runId).toBe("run"); expect(id).toBe(retainedId); order.push("stop"); return f.settle(id); }); const result = await stopAtPendingPermission({ scope: f.scope, caller, deadlineAt: Date.now() + 1000, load: async () => { order.push("load"); return f.state(); },