diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 48af3c96c25..b08d1adc712 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -421,10 +421,20 @@ both input orders. Neither policy claims the original ACP wire order of a live attempt. Old v1 receipts are not valid inputs to the new grader. Earlier paid failures retain their original definition and grade. -`paperclip.e2e.native-active-stop-settlement.v1` accepts only -`pending_permission_cancelled`: explicit-cancellation request closure with -`replayAllowed:false`, then one exact `turn.cancelled`, plus the same scoped -caller-owned native Stop acknowledgement. Missing, duplicate, foreign, failed, +Suite version 4 and `paperclip.e2e.native-active-stop-settlement.v2` accept only +`pending_permission_cancelled`: the Product harness's exact +`runtime_request.cancelled` closure (`reason: turn_terminal`, matching request, +item and turn, no answer), then one exact `turn.cancelled` (`status: cancelled`, +`error: null`), plus the same scoped caller-owned native Stop acknowledgement. +Both events must belong to the retained source/session/turn and follow all four +pre-Stop evidence rows. The harness consumes raw backend request closures and +projects its own pending-request outcome before the terminal. Raw backend-only +`provider`, `requestType`, `replayAllowed` and `providerTurnId` fields are not +required from that Product projection. Replay refusal is independently checked +through the stale public answer below. A generic terminal without the retained +request and exact acknowledged caller UUID is insufficient. The v2 settlement +receipt records this changed oracle; historical v1 receipts and failed attempts +retain their original grades. Missing, duplicate, foreign, failed, interrupted or normal terminal evidence fails. Only after this proof does the fixture attempt a stale public answer, requiring HTTP409 and an unanswerable browser card. It retains one cancelled run, issue `in_progress`, exact target diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 072f34938d9..980f33871fc 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1460,6 +1460,15 @@ Native notice ordering follows each real projector: Cursor waits for its exact command origin before emitting permission evidence; Copilot can emit the permission notice first. Both still require the exact tool origin before Stop. +Suite version 4 uses the normalized Product cancellation contract and writes +`paperclip.e2e.native-active-stop-settlement.v2`. It requires the exact request, +item and turn closure with `reason: turn_terminal`, followed by the same stream's +`turn.cancelled` with `status: cancelled` and `error: null`. The Product harness +emits these fields; raw backend-only cancellation metadata is not its contract. +The retained pending v2 receipt, caller UUID and durable scoped acknowledgement +remain mandatory. This fixes an oracle mismatch observed after a real Stop was +acknowledged; the failed attempt remains failed and needs a fresh live run. + Passage requires a canonical cancelled request and cancelled provider turn, exact caller-intent acknowledgement, an unfinished task, rejection of a later stale answer, no follow-up run, continuous target no-effect observation and diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index cf1da325a54..9fe7a778989 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1052,7 +1052,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Stop while one exact Cursor or Copilot native permission remains unanswered; require cancelled provider settlement, caller-owned acknowledgement, stale-answer refusal and independent retirement/no effects.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => ["cursor", "copilot"].includes(profile.qualificationCandidate ?? "")), environments: runnerEnvironments, tasks: nativeActiveStopTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", evidence: "paperclip.e2e.native-active-stop-settlement.v1", pendingObservation: "retained-api-before-caller-uuid-stop", normalCompletionAccepted: false, permissionPolicy: "approve-reads", lifecycle: "per_turn", remoteEvidence: "paperclip.e2e.native-active-stop-remote-retirement.v1", remoteObservationCoverage: "continuous-through-owned-process-retirement", filesystemAfterRemoteRetirementObserved: false, localObservationCoverage: "four-phases-through-cleanup", providerDeath: "not-covered" }, + definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", evidence: "paperclip.e2e.native-active-stop-settlement.v2", pendingObservation: "retained-api-before-caller-uuid-stop", normalCompletionAccepted: false, permissionPolicy: "approve-reads", lifecycle: "per_turn", remoteEvidence: "paperclip.e2e.native-active-stop-remote-retirement.v1", remoteObservationCoverage: "continuous-through-owned-process-retirement", filesystemAfterRemoteRetirementObserved: false, localObservationCoverage: "four-phases-through-cleanup", providerDeath: "not-covered" }, }, { id: "copilot-protection", label: "Copilot native protection", manualOnly: true, diff --git a/tests/runner-e2e/native-active-stop-evidence.ts b/tests/runner-e2e/native-active-stop-evidence.ts index 566baf4290f..1b7b0d22ffb 100644 --- a/tests/runner-e2e/native-active-stop-evidence.ts +++ b/tests/runner-e2e/native-active-stop-evidence.ts @@ -147,9 +147,14 @@ export function readActiveStopSettlement(input: { events: readonly unknown[]; ru const closed = proof.rows.filter(x => closures.has(x.event.eventType)), terminal = proof.rows.filter(x => terminals.has(x.event.eventType)); fail(closed.length === 1 && terminal.length === 1, "one closed request and one terminal required"); const c = closed[0]!, t = terminal[0]!, cp = rec(c.event.payload), tp = rec(t.event.payload); + // Product persists the harness projection, not the raw runnerd backend + // payload. The terminal mapper closes pending permissions with turn_terminal + // and emits {status,error}. Bind that exact boundary to the retained native + // request and caller-owned Stop below; a generic terminal is insufficient. fail(proof.stream(c.event) && proof.stream(t.event) && c.event.eventType === "runtime_request.cancelled" && cp.requestId === before.requestId && cp.turnId === before.turnId - && cp.requestKind === "permission_approval" && cp.requestType === "permission" && cp.reason === "explicit_cancellation" - && cp.provider === "acpx" && cp.itemId === proof.request.itemId && cp.replayAllowed === false && t.event.eventType === "turn.cancelled" && tp.status === "cancelled" && tp.provider === "acpx" && tp.providerTurnId === before.turnId && tp.error === null + && cp.requestKind === "permission_approval" && cp.reason === "turn_terminal" + && cp.itemId === proof.request.itemId && cp.action === undefined && cp.response === undefined && cp.replayAllowed !== true + && t.event.eventType === "turn.cancelled" && tp.status === "cancelled" && tp.error === null && c.event.sourceSeq > Math.max(before.requestSourceSeq, before.permissionSourceSeq, before.toolOriginSourceSeq, before.toolStartedSourceSeq) && t.event.sourceSeq > c.event.sourceSeq, "pending callback did not close through cancellation"); @@ -166,7 +171,7 @@ export function readActiveStopSettlement(input: { events: readonly unknown[]; ru && Array.isArray(stop.effects) && stop.effects.length === 1 && stop.effects[0] === "release_run_resources" && id(stop.intentAuditId) && id(stop.acknowledgementAuditId) && stop.intentAuditId !== stop.acknowledgementAuditId, "same-scope caller Stop acknowledgement missing"); - return { schema: "paperclip.e2e.native-active-stop-settlement.v1", pending: before, dispatchMonotonicNs: input.dispatchMonotonicNs, + return { schema: "paperclip.e2e.native-active-stop-settlement.v2", pending: before, dispatchMonotonicNs: input.dispatchMonotonicNs, branch: "pending_permission_cancelled" as const, closedRequestSourceSeq: c.event.sourceSeq, terminalSourceSeq: t.event.sourceSeq, closedRequestRowSha256: hash(c.row), terminalRowSha256: hash(t.row), intentId: stop.intentId as string, intentAuditId: stop.intentAuditId as string, acknowledgementAuditId: stop.acknowledgementAuditId as string, diff --git a/tests/runner-e2e/native-active-stop.test.ts b/tests/runner-e2e/native-active-stop.test.ts index f6317468b6e..1403e9c8309 100644 --- a/tests/runner-e2e/native-active-stop.test.ts +++ b/tests/runner-e2e/native-active-stop.test.ts @@ -8,6 +8,9 @@ import { createCursorToolEvidence } from "../../packages/paperclip-runner/src/dr import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js"; import type { CanonicalProviderEvent } from "../../packages/paperclip-runner/src/provider-events.js"; +import { CodexSessionState } from "../../packages/paperclip-runner/src/drivers/codex/codex-session-state.js"; +import { mapTerminalTurn } from "../../packages/paperclip-runner/src/drivers/codex/codex-session-terminal.js"; + const caller = readActiveStopCaller({ deploymentMode: "local_trusted" }, { session: { userId: "local-board", id: "paperclip:local_implicit:local-board" } }); const cancellationRequestId = "11111111-2222-4333-8444-555555555555"; type Row = Record; @@ -26,7 +29,7 @@ function fixture(provider: ActiveStopProvider = "copilot") { notice(1, "tool", { status: "pending", ...operation }), row(2, "tool.execution.started", { schema: "paperclip.tool.execution.v1", executionId: "tool", transport: "builtin", status: "running", ...operation }), notice(3, "permission_requested", { requestId: "request", declineOffered: true, ...operation }), - row(4, "runtime_request.created", { request: { schema: "paperclip.runtime_request.v2", requestKind: "permission_approval", type: "permission", status: "pending", requestId: "request", turnId: "turn", itemId: "tool", + row(4, "runtime_request.created", { request: { schema: "paperclip.runtime_request.v2", requestKind: "permission_approval", type: "permission", status: "pending", requestId: "request", turnId: "turn", itemId: "item-run", method: "item/commandExecution/requestApproval", details: { toolCallId: "tool" }, origin: { adapter: "acpx-runtime-sidecar", provider, method: "session/request_permission" } } }), ]; const run: Row = { id: "run", companyId: "company", nativeIssueId: "issue", runtimeMode: "native", status: "running", resultJson: {} }; @@ -34,8 +37,17 @@ function fixture(provider: ActiveStopProvider = "copilot") { const state = () => ({ events, run, issue }); const pending = () => observeActiveStopPending({ ...state(), scope, caller, cancellationRequestId }); const settle = (requestId = cancellationRequestId) => { - events.push(row(5, "runtime_request.cancelled", { provider: "acpx", requestId: "request", turnId: "turn", requestKind: "permission_approval", requestType: "permission", itemId: "tool", reason: "explicit_cancellation", replayAllowed: false }), - row(6, "turn.cancelled", { provider: "acpx", providerTurnId: "turn", status: "cancelled", error: null })); + // Exercise the actual Product terminal and pending-request producers. Only + // the state storage/emitter and callback are test doubles; no provider runs. + const request = events.find(event => event.eventType === "runtime_request.created")!.payload.prpEvent.payload.request; + const state = { + terminalTurns: new Map(), workspaceChangesByTurn: new Map(), result: null, + conversationMode: "direct", activeTurnId: "turn", turnStarted: true, + pendingRuntimeRequestMap: new Map([[request.requestId, { request, settle: vi.fn() }]]), + cancelPendingRequests: CodexSessionState.prototype.cancelPendingRequests, + emit: (eventType: string, value: Row) => events.push(row(events.length + 1, eventType, value)), + } as unknown as CodexSessionState; + mapTerminalTurn(state, { id: "turn", status: "cancelled", error: null }, "turn"); run.status = "cancelled"; run.resultJson = { startupCancellation: { cancellationRequestId: requestId, requestedBy: { type: "board", userId: "local-board" } }, nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: `native-cancellation:${requestId}`, companyId: "company", runId: "run", issueId: "issue", scope: "run", dispatched: true, dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", @@ -203,10 +215,20 @@ describe("definitely active native permission Stop", () => { it.each(["cursor", "copilot"] as const)("binds %s's unanswered callback to cancelled provider settlement and caller-owned Stop", provider => { const f = fixture(provider), pending = f.pending(); f.settle(); expect(readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() })).toMatchObject({ - schema: "paperclip.e2e.native-active-stop-settlement.v1", branch: "pending_permission_cancelled", normalCompletionAccepted: false, taskStillInProgress: true, replayAllowed: false, + schema: "paperclip.e2e.native-active-stop-settlement.v2", branch: "pending_permission_cancelled", normalCompletionAccepted: false, taskStillInProgress: true, replayAllowed: false, pending: { normalizedSessionId: "normalized-session", nativeSessionId: "native-session", cancellationRequestId }, }); }); + it("matches the retained Product closure shape with distinct native and item identities", () => { + const { f, pending, read } = settled(); + expect(pending.toolCallId).toBe("tool"); + // Content-free shape captured from the failed Product attempt. These exact + // payloads come from mapTerminalTurn -> cancelPendingRequests above. + expect(payload(f.events[4]!)).toEqual({ requestId: "request", requestKind: "permission_approval", + turnId: "turn", itemId: "item-run", reason: "turn_terminal" }); + expect(payload(f.events[5]!)).toEqual({ status: "cancelled", error: null }); + expect(read().schema).toBe("paperclip.e2e.native-active-stop-settlement.v2"); + }); it("admits only a fully attested remote bootstrap read completed before the pending operation", () => { const f = fixture(), actionFile = `.paperclip-eval-action-${"a".repeat(36)}.txt`; const readTargetSha256 = `sha256:${createHash("sha256").update(actionFile).digest("hex")}`; @@ -249,6 +271,13 @@ describe("definitely active native permission Stop", () => { ["foreign request", ({ f }: ReturnType) => { payload(f.events[4]!).requestId = "other"; }], ["expired rather than cancelled", ({ f }: ReturnType) => { f.events[4]!.eventType = frame(f.events[4]!).eventType = "runtime_request.expired"; }], ["provider death", ({ f }: ReturnType) => { payload(f.events[4]!).reason = "provider_process_lost"; }], + ["raw backend closure is not the Product contract", ({ f }: ReturnType) => { payload(f.events[4]!).reason = "explicit_cancellation"; }], + ["answered closure", ({ f }: ReturnType) => { payload(f.events[4]!).action = "accept"; }], + ["resolved request", ({ f }: ReturnType) => { f.events[4]!.eventType = frame(f.events[4]!).eventType = "runtime_request.resolved"; }], + ["duplicate closure", ({ f }: ReturnType) => { f.events.push(f.row(7, "runtime_request.cancelled", payload(f.events[4]!))); }], + ["foreign closure turn", ({ f }: ReturnType) => { payload(f.events[4]!).turnId = "other"; }], + ["missing acknowledgement", ({ f }: ReturnType) => { delete f.run.resultJson.nativeCancellation.acknowledgementAuditId; }], + ["unacknowledged intent", ({ f }: ReturnType) => { f.run.resultJson.nativeCancellation.dispatchState = "pending"; }], ["replay permitted", ({ f }: ReturnType) => { payload(f.events[4]!).replayAllowed = true; }], ["tampered pending row", ({ f }: ReturnType) => { payload(f.events[3]!).request.itemId = "changed"; }], ["missing pending receipt", (s: ReturnType) => { s.pending.requestRowSha256 = ""; }], @@ -345,8 +374,8 @@ describe("explicit active Stop discovery", () => { expect(cells).toHaveLength(4); expect(suite.manualOnly).toBe(true); expect(cells.map(e => `${e.profile.qualificationCandidate}/${e.environment.id}`).sort()).toEqual(["copilot/daytona", "copilot/local", "cursor/daytona", "cursor/local"]); expect(cells.every(e => e.task.expectedRunCount === 1 && e.task.flow === "native_active_stop" && e.task.expectedTerminalState?.run === "cancelled")).toBe(true); - expect(suite.definitionMetadata).toMatchObject({ version: 3, normalCompletionAccepted: false, providerDeath: "not-covered" }); - expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 2 } })); + expect(suite.definitionMetadata).toMatchObject({ version: 4, evidence: "paperclip.e2e.native-active-stop-settlement.v2", normalCompletionAccepted: false, providerDeath: "not-covered" }); + expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 3 } })); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, normalCompletionAccepted: true } })); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); });