From a6a5417cd9cb4e664fecf8d7fd6cc3d8ed9b17e9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 14:42:10 -0500 Subject: [PATCH] test(runner-e2e): qualify Stop at pending native permission Add explicit Cursor and Copilot local/Daytona cells that retain an unanswered callback before a caller-correlated Stop, require cancelled provider settlement and stale-answer rejection, and independently verify no effects through owned retirement. Normal completion and provider death do not satisfy this active-work oracle. Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 31 ++++ tests/runner-e2e/README.md | 26 +++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 8 + .../runner-e2e/native-active-stop-evidence.ts | 161 +++++++++++++++++ tests/runner-e2e/native-active-stop-flow.ts | 167 +++++++++++++++++ tests/runner-e2e/native-active-stop-tasks.ts | 10 ++ tests/runner-e2e/native-active-stop.test.ts | 169 ++++++++++++++++++ tests/runner-e2e/prerequisites.ts | 2 +- tests/runner-e2e/runner.spec.ts | 14 +- tests/runner-e2e/types.ts | 1 + 11 files changed, 589 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/native-active-stop-evidence.ts create mode 100644 tests/runner-e2e/native-active-stop-flow.ts create mode 100644 tests/runner-e2e/native-active-stop-tasks.ts create mode 100644 tests/runner-e2e/native-active-stop.test.ts diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 920387cd18e..6328a278013 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -391,3 +391,34 @@ the response and final `paperclip.e2e.copilot-denial-settlement.v3` receipt. It refuses a non-running controller or existing Stop marker before dispatch. The completed-provider branch still requires a running controller that this request can stop; it does not accept a no-op Stop of an already terminal run. + +## Definitely-active native Stop + +`native-active-stop` / `pending-permission-stop` has four explicit-only cells: +Cursor and Copilot, each local and Daytona. Its `native_active_stop` flow retains +`paperclip.e2e.native-active-stop-pending.v1` from the public API while exactly one +native permission remains pending, the exact controller run is running, and no +Stop or answer marker exists. The receipt independently binds native session, +normalized session, turn, source instance, request/tool IDs, source sequences and +canonical row hashes. An awaited artifact write and fresh pending reread precede +Stop dispatch; process-monotonic timestamps describe only these local observer +boundaries. Remote provider clocks and database transaction timestamps never +establish that ordering. The atomic caller UUID fence rejects an earlier racing +Stop instead of borrowing its acknowledgement. + +`paperclip.e2e.native-active-stop-settlement.v1` accepts only +`pending_permission_cancelled`: explicit-cancellation request closure with +`replayAllowed:false`, then one exact `turn.cancelled`, plus the same scoped +caller-owned native Stop acknowledgement. Missing, duplicate, foreign, failed, +interrupted or normal terminal evidence fails. Only after this proof does the +fixture attempt a stale public answer, requiring HTTP409 and an unanswerable +browser card. It retains one cancelled run, issue `in_progress`, exact target +absence through continuous observation, and all observed owned descendants +retired. Remote proof is sealed before environment teardown; unproven cleanup +fails independently. Only fully attested bootstrap reads may precede the one +tested operation; alternate operations or attempts are rejected. + +The existing `copilot-protection` denial remains distinct: rejecting a permission +before Stop does not exercise this pending-callback boundary. This new suite has +pure calibration and wiring tests, not a paid qualification result. Provider +process death is not simulated by substituting the chat runner-worker crash hook. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 339a384d0bd..88954f61e79 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1447,3 +1447,29 @@ pre-Stop observation to the public cancel request and its exact durable native intent. It rejects prior startup/Stop markers before dispatch and a competing request that wins after the last read. The server atomically reserves the caller identity; the fixture does not infer ownership from timestamps or HTTP success. + +## Stop during an unanswered native permission (explicit only) + +`--suite native-active-stop --task pending-permission-stop` selects Cursor and +Copilot on local or Daytona, one run per cell. These four cells are excluded +from `--all`. They use `approve-reads`, per-turn lifecycle and Cursor Agent mode; +provider timeout is 120s and the attempt budget is 300s. The fixture asks for one +exact native write, observes its unanswered permission card, then sends the +public Stop request with a freshly retained caller UUID. It never denies or +approves that callback before Stop. + +Passage requires a canonical cancelled request and cancelled provider turn, +exact caller-intent acknowledgement, an unfinished task, rejection of a later +stale answer, no follow-up run, continuous target no-effect observation and +owned process retirement. Normal completion and interrupted/failed turns do +not qualify this case. Native permission cards use runtime requests; ordinary +issue-interaction rows are not substituted for their authority. Pending and final +screenshots, `native-active-stop-pending.json`, `native-active-stop-settlement.json` +and cleanup evidence retain the boundaries. Daytona also requires the exact +owned lease and sealed remote observer proof before sandbox deletion. + +This suite adds an active-work cancellation oracle; it does not reinterpret +older denial or cancellation results. Current live qualification is pending. +Provider death during a pending callback remains a separate uncovered case: +the existing chat worker-crash hook targets a runner worker and cannot establish +safe ownership of the native provider process in both environments. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index cd88bd7f3ba..6b2078d7b29 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -91,10 +91,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 8, 7, 4, 6, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, + 8, 7, 4, 4, 6, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(432); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(432); + expect(validateRunnerCatalog()).toHaveLength(436); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(436); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 207b7565580..96fd15eac46 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,4 @@ +import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { cursorNativeTasks } from "./cursor-native-cases.js"; import { copilotProtectionTasks } from "./copilot-protection-tasks.js"; import { piNativeTasks } from "./pi-native-cases.js"; @@ -1046,6 +1047,13 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial"], definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer" }, }, + { + id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, + description: "Stop while one exact Cursor or Copilot native permission remains unanswered; require cancelled provider settlement, caller-owned acknowledgement, stale-answer refusal and independent retirement/no effects.", + groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => ["cursor", "copilot"].includes(profile.qualificationCandidate ?? "")), + environments: runnerEnvironments, tasks: nativeActiveStopTasks, expectedMatrixSize: 4, + definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", evidence: "paperclip.e2e.native-active-stop-settlement.v1", pendingObservation: "retained-api-before-caller-uuid-stop", normalCompletionAccepted: false, permissionPolicy: "approve-reads", lifecycle: "per_turn", remoteEvidence: "owned-lease-sealed-observer", providerDeath: "not-covered" }, + }, { id: "copilot-protection", label: "Copilot native protection", manualOnly: true, description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", diff --git a/tests/runner-e2e/native-active-stop-evidence.ts b/tests/runner-e2e/native-active-stop-evidence.ts new file mode 100644 index 00000000000..ad746983064 --- /dev/null +++ b/tests/runner-e2e/native-active-stop-evidence.ts @@ -0,0 +1,161 @@ +import { createHash } from "node:crypto"; +import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; +import { hasAcpxNativeOrigin } from "./acpx-native-origin.js"; +import { readCopilotToolEvidence } from "./copilot-evidence.js"; +import { readCursorToolEvidence } from "./cursor-native-evidence.js"; +import { bootstrapReadExecutionId, withoutProvenBootstrapReads, type BootstrapReadProof } from "./native-bootstrap-read-proof.js"; + +type Row = Record; +export type ActiveStopProvider = "cursor" | "copilot"; +export interface ActiveStopScope { provider: ActiveStopProvider; companyId: string; issueId: string; runId: string; target: string; commandSha256?: string } +export interface ActiveStopPending { + schema: "paperclip.e2e.native-active-stop-pending.v1"; + scope: ActiveStopScope; requestId: string; toolCallId: string; nativeSessionId: string; turnId: string; + normalizedSessionId: string; sourceInstanceId: string; requestSourceSeq: number; permissionSourceSeq: number; + requestRowSha256: string; permissionRowSha256: string; cancellationRequestId: string; observedMonotonicNs: string; +} +const rec = (v: unknown): Row => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Row : {}; +const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]"); +const uuid = (v: unknown): v is string => typeof v === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/u.test(v); +const hash = (v: unknown) => `sha256:${createHash("sha256").update(canonicalJson(v)).digest("hex")}`; +function fail(condition: unknown, reason: string): asserts condition { if (!condition) throw new Error(`Native active Stop: ${reason}`); } +function canonicalRows(events: readonly unknown[], scope: ActiveStopScope) { + fail(events.length <= 20_000 && events.length > 0, "bounded durable evidence required"); + const rows = events.map(rec).filter(row => rec(row.payload).prpEvent !== undefined); + const seen = new Set(), source = new Set(); + for (const row of rows) { + const e = rec(rec(row.payload).prpEvent); + fail(row.companyId === scope.companyId && row.runId === scope.runId && row.protocolSchemaVersion === 1 + && e.schema === "paperclip.prp.event.v1" && e.schemaVersion === 1 && e.sourceKind === "runner" && e.runId === scope.runId + && e.eventType === row.eventType && Number.isSafeInteger(row.seq) && row.seq > 0 && !seen.has(row.seq) + && id(e.sourceInstanceId) && Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > 0 + && e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !source.has(e.sourceEventId), "invalid/duplicate/foreign canonical row"); + seen.add(row.seq); source.add(e.sourceEventId); + } + const ordered = [...rows].sort((a, b) => a.seq - b.seq); + const lastBySource = new Map(); + for (const row of ordered) { + const e = rec(rec(row.payload).prpEvent), previous = lastBySource.get(e.sourceInstanceId) ?? 0; + fail(e.sourceSeq > previous, "durable source sequence reordered"); + lastBySource.set(e.sourceInstanceId, e.sourceSeq); + } + return ordered.map(row => ({ row, event: rec(rec(row.payload).prpEvent) })); +} +const terminals = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]); +const closures = new Set(["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]); +function origin(events: readonly unknown[], scope: ActiveStopScope, bootstrap?: BootstrapReadProof) { + fail([scope.companyId, scope.issueId, scope.runId, scope.target].every(id) && ["cursor", "copilot"].includes(scope.provider), "exact scope required"); + const rows = canonicalRows(events, scope); + const notices = scope.provider === "cursor" ? readCursorToolEvidence(events, scope.runId) : readCopilotToolEvidence(events, scope.runId); + const requests = notices.filter(n => n.stage === "permission_requested"); + fail(requests.length === 1, "one native permission required"); + const notice = requests[0]!; + fail(id(notice.requestId) && notice.declineOffered === true && (scope.provider === "cursor" + ? notice.operation === "execute" && /^sha256:[a-f0-9]{64}$/u.test(scope.commandSha256 ?? "") && notice.commandSha256 === scope.commandSha256 + : notice.operation === "edit" && "target" in notice && notice.target === scope.target), "exact native operation required"); + const permission = rows.find(x => x.row.seq === notice.seq); + const created = rows.filter(x => x.event.eventType === "runtime_request.created"); + fail(permission && created.length === 1, "one durable native card required"); + const request = rec(rec(created[0]!.event.payload).request), p = permission.event; + fail(request.schema === "paperclip.runtime_request.v2" && request.type === "permission" && request.requestKind === "permission_approval" + && request.status === "pending" && request.requestId === notice.requestId && request.turnId === notice.turnId + && rec(request.details).toolCallId === notice.toolCallId && hasAcpxNativeOrigin(request.origin, scope.provider, "session/request_permission") + && id(p.normalizedSessionId) && p.turnId === notice.turnId && created[0]!.event.turnId === notice.turnId + && created[0]!.event.normalizedSessionId === p.normalizedSessionId && created[0]!.event.sourceInstanceId === p.sourceInstanceId, + "native notice/card identity mismatch"); + const stream = (event: Row) => event.turnId === notice.turnId && event.normalizedSessionId === p.normalizedSessionId && event.sourceInstanceId === p.sourceInstanceId; + fail(rows.filter(x => x.event.turnId != null).every(x => stream(x.event)), "foreign turn/session/source"); + const tool = notices.find(n => n.stage === "tool" && n.toolCallId === notice.toolCallId); + fail(tool, "native operation origin missing"); + const selected = withoutProvenBootstrapReads(notices, tool, bootstrap); + fail(selected.every(n => n.runId === notice.runId && n.sessionId === notice.sessionId && n.turnId === notice.turnId + && n.toolCallId === notice.toolCallId && (n.requestId === undefined || n.requestId === notice.requestId) + && (n.operation === undefined || n.operation === notice.operation) + && (!("target" in n) || n.target === undefined || n.target === scope.target) + && (n.commandSha256 === undefined || n.commandSha256 === scope.commandSha256) + && n.status !== "completed" && (n.stage !== "permission_delivered" || n.outcome === "cancel")), "extra/replayed/answered native operation"); + const tools = selected.filter(n => n.stage === "tool"); + fail(tools.filter(n => n.status === "pending").length === 1 + && tools.filter(n => n.status === "failed").length <= 1 + && tools[0]!.status === "pending" && tools[0]!.seq < notice.seq + && selected.filter(n => n.stage === "permission_delivered").length <= 1, "duplicate native operation lifecycle"); + const executions = rows.filter(x => x.event.eventType.startsWith("tool.execution.") && rec(x.event.payload).executionId === bootstrapReadExecutionId(notice.toolCallId)); + fail(executions.filter(x => x.event.eventType === "tool.execution.started").length === 1 + && executions.filter(x => x.event.eventType === "tool.execution.completed").length <= 1 + && executions.every(x => rec(x.event.payload).schema === "paperclip.tool.execution.v1" && rec(x.event.payload).transport === "builtin" + && ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"].includes(x.event.eventType) + && rec(x.event.payload).status === (x.event.eventType === "tool.execution.completed" ? "failed" : "running")) + && executions.find(x => x.event.eventType === "tool.execution.started")!.event.sourceSeq < created[0]!.event.sourceSeq, "duplicate canonical operation lifecycle"); + const allowed = new Set(notices.filter(n => n.stage === "tool").map(n => bootstrapReadExecutionId(n.toolCallId))); + fail(rows.filter(x => x.event.eventType.startsWith("tool.execution.")).every(x => allowed.has(rec(x.event.payload).executionId)), "unattributed operation"); + fail(rows.filter(x => x.event.eventType === "tool.execution.completed" && rec(x.event.payload).executionId === bootstrapReadExecutionId(notice.toolCallId)) + .every(x => rec(x.event.payload).status !== "completed"), "target operation completed"); + return { rows, notice, permission, created: created[0]!, request, stream }; +} +export function observeActiveStopPending(input: { events: readonly unknown[]; run: Row; issue: Row; scope: ActiveStopScope; cancellationRequestId: string; bootstrap?: BootstrapReadProof }): ActiveStopPending { + const { run, issue, scope } = input, proof = origin(input.events, scope, input.bootstrap); + fail(uuid(input.cancellationRequestId) && run.id === scope.runId && run.companyId === scope.companyId && run.nativeIssueId === scope.issueId + && run.status === "running" && run.runtimeMode === "native" && issue.id === scope.issueId && issue.companyId === scope.companyId && issue.status === "in_progress" + && run.resultJson?.startupCancellation == null && run.resultJson?.nativeCancellation == null, "run is not fresh active work"); + fail(!(scope.provider === "cursor" ? readCursorToolEvidence(input.events, scope.runId) : readCopilotToolEvidence(input.events, scope.runId)).some(n => n.stage === "permission_delivered"), "permission already answered"); + fail(!proof.rows.some(x => terminals.has(x.event.eventType) || closures.has(x.event.eventType)), "request or provider already settled"); + fail(!proof.rows.some(x => x.event.eventType === "tool.execution.completed" && rec(x.event.payload).executionId === bootstrapReadExecutionId(proof.notice.toolCallId)), "operation already ended"); + return { schema: "paperclip.e2e.native-active-stop-pending.v1", scope, requestId: proof.notice.requestId!, toolCallId: proof.notice.toolCallId, + nativeSessionId: proof.notice.sessionId, turnId: proof.notice.turnId, normalizedSessionId: proof.permission.event.normalizedSessionId, + sourceInstanceId: proof.permission.event.sourceInstanceId, requestSourceSeq: proof.created.event.sourceSeq, permissionSourceSeq: proof.permission.event.sourceSeq, + requestRowSha256: hash(proof.created.row), permissionRowSha256: hash(proof.permission.row), cancellationRequestId: input.cancellationRequestId, + observedMonotonicNs: process.hrtime.bigint().toString() }; +} +export function readActiveStopSettlement(input: { events: readonly unknown[]; run: Row; issue: Row; pending: ActiveStopPending; dispatchMonotonicNs: string; bootstrap?: BootstrapReadProof }) { + const before = input.pending, { run, issue } = input; + fail(before?.schema === "paperclip.e2e.native-active-stop-pending.v1" && uuid(before.cancellationRequestId) + && /^[1-9][0-9]{0,29}$/u.test(before.observedMonotonicNs) && /^[1-9][0-9]{0,29}$/u.test(input.dispatchMonotonicNs) + && BigInt(before.observedMonotonicNs) < BigInt(input.dispatchMonotonicNs), "pre-dispatch pending observation missing"); + const proof = origin(input.events, before.scope, input.bootstrap); + fail(hash(proof.created.row) === before.requestRowSha256 && hash(proof.permission.row) === before.permissionRowSha256 + && proof.notice.sessionId === before.nativeSessionId && proof.notice.turnId === before.turnId && proof.notice.toolCallId === before.toolCallId + && proof.notice.requestId === before.requestId && proof.permission.event.normalizedSessionId === before.normalizedSessionId + && proof.permission.event.sourceInstanceId === before.sourceInstanceId && proof.created.event.sourceSeq === before.requestSourceSeq + && proof.permission.event.sourceSeq === before.permissionSourceSeq, "observed pending request changed"); + const closed = proof.rows.filter(x => closures.has(x.event.eventType)), terminal = proof.rows.filter(x => terminals.has(x.event.eventType)); + fail(closed.length === 1 && terminal.length === 1, "one closed request and one terminal required"); + const c = closed[0]!, t = terminal[0]!, cp = rec(c.event.payload), tp = rec(t.event.payload); + fail(c.event.eventType === "runtime_request.cancelled" && cp.requestId === before.requestId && cp.turnId === before.turnId + && cp.requestKind === "permission_approval" && cp.requestType === "permission" && cp.reason === "explicit_cancellation" + && cp.provider === "acpx" && cp.itemId === proof.request.itemId && cp.replayAllowed === false && t.event.eventType === "turn.cancelled" && tp.status === "cancelled" && tp.provider === "acpx" && tp.providerTurnId === before.turnId && tp.error === null + && c.event.sourceSeq > Math.max(before.requestSourceSeq, before.permissionSourceSeq) && t.event.sourceSeq > c.event.sourceSeq, + "pending callback did not close through cancellation"); + fail(proof.rows.filter(x => x.event.eventType.startsWith("tool.execution.") + || (x.event.eventType === "provider.notice.recorded" && ["cursor_tool_evidence_v1", "copilot_tool_evidence_v1"].includes(rec(x.event.payload).category))) + .every(x => x.event.sourceSeq < t.event.sourceSeq), "native operation after provider cancellation"); + const stop = rec(rec(run.resultJson).nativeCancellation), startup = rec(rec(run.resultJson).startupCancellation); + fail(run.id === before.scope.runId && run.companyId === before.scope.companyId && run.nativeIssueId === before.scope.issueId && run.runtimeMode === "native" && run.status === "cancelled" + && issue.id === before.scope.issueId && issue.companyId === before.scope.companyId && issue.status === "in_progress" + && startup.cancellationRequestId === before.cancellationRequestId && stop.intentId === `native-cancellation:${before.cancellationRequestId}` + && stop.schema === "paperclip.native-cancellation.v1" && stop.companyId === before.scope.companyId && stop.runId === run.id && stop.issueId === issue.id + && stop.scope === "run" && stop.dispatched === true && stop.dispatchState === "acknowledged" && stop.reasonCode === "cancellation_run_only" + && Array.isArray(stop.effects) && stop.effects.length === 1 && stop.effects[0] === "release_run_resources" + && id(stop.intentAuditId) && id(stop.acknowledgementAuditId) && stop.intentAuditId !== stop.acknowledgementAuditId, + "same-scope caller Stop acknowledgement missing"); + return { schema: "paperclip.e2e.native-active-stop-settlement.v1", pending: before, dispatchMonotonicNs: input.dispatchMonotonicNs, + branch: "pending_permission_cancelled" as const, closedRequestSourceSeq: c.event.sourceSeq, terminalSourceSeq: t.event.sourceSeq, + closedRequestRowSha256: hash(c.row), terminalRowSha256: hash(t.row), intentId: stop.intentId as string, + intentAuditId: stop.intentAuditId as string, acknowledgementAuditId: stop.acknowledgementAuditId as string, + taskStillInProgress: true, normalCompletionAccepted: false, replayAllowed: false }; +} + +/** Read-only watcher/process receipts are mandatory through teardown, even if + * canonical cancellation already passed. A missing phase is not no-effect proof. */ +export function assertActiveStopRetirement(input: { + completed: boolean; identityChanged: boolean; + processes: { captured: boolean; live: readonly number[] }; + watcher: { complete: boolean; targetMutationCount: number }; + samples: ReadonlyArray<{ phase: string; absent: boolean }>; +}) { + fail(input.completed && !input.identityChanged && input.processes.captured + && input.processes.live.length === 0 && input.watcher.complete && input.watcher.targetMutationCount === 0, + "retirement or continuous no-effect proof is incomplete"); + fail(input.samples.length === 4 && input.samples.every(s => s.absent === true) + && input.samples.map(s => s.phase).join(",") === "before-request,pending,after-stop,after-cleanup", + "causal target observation missing or changed"); +} diff --git a/tests/runner-e2e/native-active-stop-flow.ts b/tests/runner-e2e/native-active-stop-flow.ts new file mode 100644 index 00000000000..07c337aa905 --- /dev/null +++ b/tests/runner-e2e/native-active-stop-flow.ts @@ -0,0 +1,167 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { collectRunEvents } from "./run-observations.js"; +import { createTaskThroughUi } from "./user-actions.js"; +import { createDeniedTargetFixture, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; +import { assertCopilotRemoteRetirement, copilotRemoteDeniedSample, prepareCopilotRemoteAction, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot } from "./copilot-protection-evidence.js"; +import { cursorDeniedCommand } from "./cursor-native-evidence.js"; +import { assertActiveStopRetirement, observeActiveStopPending, readActiveStopSettlement, type ActiveStopPending, type ActiveStopScope } from "./native-active-stop-evidence.js"; +import type { BootstrapReadProof } from "./native-bootstrap-read-proof.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import type { MatrixExecution } from "./types.js"; +type Row = Record; +type Check = { id: string; passed: boolean; detail: string }; +export interface ActiveStopState { run: Row; issue: Row; events: readonly unknown[] } +/** Retain a real pending API observation before the only mutating request. + * Never answer first, wait for natural completion, or infer order from clocks. */ +export async function stopAtPendingPermission(input: { + scope: ActiveStopScope; bootstrap?: BootstrapReadProof; deadlineAt: number; + load(): Promise; + retain(receipt: ActiveStopPending): Promise; + stop(runId: string, cancellationRequestId: string): Promise; +}) { + const cancellationRequestId = randomUUID(); + const state = await input.load(); + const pending = observeActiveStopPending({ ...state, scope: input.scope, cancellationRequestId, bootstrap: input.bootstrap }); + await input.retain(pending); + const fresh = observeActiveStopPending({ ...await input.load(), scope: input.scope, cancellationRequestId, bootstrap: input.bootstrap }); + if (fresh.requestRowSha256 !== pending.requestRowSha256 || fresh.permissionRowSha256 !== pending.permissionRowSha256 || Date.now() >= input.deadlineAt) throw new Error("Native active Stop pending boundary changed or expired"); + const dispatchMonotonicNs = process.hrtime.bigint().toString(); + const stopped = await input.stop(input.scope.runId, cancellationRequestId); + if (stopped.id !== input.scope.runId || stopped.resultJson?.nativeCancellation?.intentId !== `native-cancellation:${cancellationRequestId}`) throw new Error("Native active Stop response has a foreign intent"); + const label = "cancelled unanswered native permission"; + const final = await pollUntil({ label, deadlineAt: input.deadlineAt, intervalMs: 200, load: async () => { + const value = await input.load(); + if (["succeeded", "failed", "timed_out"].includes(value.run.status)) throw new Error(`Stopped waiting for ${label}: unexpected run terminal`); + return value; + }, accept: value => { readActiveStopSettlement({ ...value, pending, dispatchMonotonicNs, bootstrap: input.bootstrap }); return true; } }); + return { pending, dispatchMonotonicNs, settlement: readActiveStopSettlement({ ...final, pending, dispatchMonotonicNs, bootstrap: input.bootstrap }) }; +} +export async function runNativeActiveStopFlow(input: { + page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; workspacePath: string; deadlineAt: number; + remoteBootstrap?: CopilotRemoteBootstrap; + registerCleanupAssertion(callback: () => Promise): void; + registerBeforeEnvironmentTeardownAssertion(callback: () => Promise): void; + observe(issue: Row, runs: Row[]): void; + capture(id: string, label: string, file: string): Promise; + evidence(name: string, value: unknown): Promise; +}) { + const { api, page, fixtures, execution, nonce } = input, provider = execution.profile.qualificationCandidate; + if ((provider !== "cursor" && provider !== "copilot") || execution.task.id !== "pending-permission-stop") throw new Error("Unknown native active Stop case"); + const remote = execution.environment.id === "daytona"; + if ((!remote && execution.environment.id !== "local") || (remote && !input.remoteBootstrap)) throw new Error("Active Stop requires an isolated admitted environment"); + const checks: Check[] = []; let issue: Row = {}, runs: Row[] = [], events: Row[] = []; + const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); }; + const name = `active-stop-${nonce}.txt`, local = remote ? undefined : await createDeniedTargetFixture(input.workspacePath, name); + const target = local?.targetRelativePath ?? name; + let command = provider === "cursor" && local ? cursorDeniedCommand(local.targetPath) : undefined; + const observer = remote ? undefined : observeRunProcesses(); + let processes: { captured: boolean; live: number[] } = { captured: false, live: [] }; + let processIdentity: string | undefined, processError = false; + const samples: Array<{ phase: string; absent: boolean }> = []; + let fixture: CopilotRemoteFixture | undefined, baseline: CopilotRemoteSnapshot | undefined, sealed: CopilotRemoteSnapshot | undefined; + let completed: Awaited> | undefined; + const observeProcesses = () => { + const run = runs[0], authority = run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined; + if (authority) { const key = JSON.stringify(authority); if (processIdentity && processIdentity !== key) processError = true; processIdentity ??= key; } + if (observer) processes = observer.sample(authority); + }; + const load = async (): Promise => { + if (issue.id) issue = await api.get(`/api/issues/${issue.id}`); + const list = await api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`); + runs = await Promise.all(list.map(r => api.get(`/api/heartbeat-runs/${r.id}`))); + if (runs.length > 1) throw new Error("Native active Stop dispatched an extra run"); + events = runs[0] ? await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${runs[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : []; + observeProcesses(); input.observe(issue, runs); return { run: runs[0] ?? {}, issue, events }; + }; + const sample = async (phase: string) => { + let value: boolean; + if (remote) { + if (!fixture || !baseline) throw new Error("Missing remote active Stop observer"); + const snap = sealed ?? await fixture.snapshot(phase); + value = !copilotRemoteDeniedSample(snap, baseline, target, "pending").exists; + await input.evidence(`active-stop-${phase}-remote.json`, snap); + } else value = !await exists(local!.targetPath); + samples.push({ phase, absent: value }); check(`no-effect-${phase}`, value, "Exact target remains absent at the causal observation boundary"); + }; + const scope = (): ActiveStopScope => ({ provider, companyId: fixtures.company.id, issueId: issue.id, runId: runs[0]!.id, target, ...(command ? { commandSha256: command.commandSha256 } : {}) }); + const bootstrap = (): BootstrapReadProof | undefined => fixture ? { actionFile: fixture.actionFile, events } : undefined; + const cleanup = async () => { + const receipt: Check[] = []; + try { + await load(); + if (remote) { + if (!fixture || !baseline) throw new Error("Remote observer was never armed"); + sealed ??= await fixture.finish(); assertCopilotRemoteRetirement(sealed, baseline); processes = sealed.processes; + } else if (processes.captured && processes.live.length) { + await pollUntil({ label: "active Stop provider retirement", deadlineAt: Date.now() + 5000, intervalMs: 100, load: async () => { observeProcesses(); return processes; }, accept: p => p.live.length === 0 }); + } + await sample("after-cleanup"); + const watch = remote ? sealed!.watcher : local!.watcher.finish(); + assertActiveStopRetirement({ completed: Boolean(completed), identityChanged: processError, processes, watcher: watch, samples }); + if (!completed) throw new Error("Missing active Stop settlement"); + readActiveStopSettlement({ ...await load(), ...completed, bootstrap: bootstrap() }); + receipt.push({ id: "active-stop-retired-no-effects", passed: true, detail: "Exact run root/descendants retired; unanswered target remained absent through continuous observation" }); + return receipt; + } finally { + if (timer) clearInterval(timer); + try { + await input.evidence("native-active-stop-cleanup.json", { processes, processError, samples, sealed, watcher: local?.watcher.finish(), checks: receipt }); + } finally { + if (remote) await fixture?.close(); + } + } + }; + const timer = observer ? setInterval(() => { try { observeProcesses(); } catch { processError = true; } }, 100) : undefined; + (remote ? input.registerBeforeEnvironmentTeardownAssertion : input.registerCleanupAssertion)(cleanup); + const prompt = () => provider === "cursor" + ? `${execution.task.buildPrompt(nonce)}\nUse this exact native shell command once:\n${command!.command}` + : `${execution.task.buildPrompt(nonce)}\nUse the native edit tool once to create exactly ${target} with content MUST_NOT_EXIST. Do not use a shell command.`; + try { + const agent = await api.get(`/api/agents/${fixtures.agent.id}`); + const configured = await api.patch(`/api/agents/${fixtures.agent.id}`, { adapterConfig: { ...agent.adapterConfig, acpxPermissionMode: "approve-reads", ...(provider === "cursor" ? { acpxSessionMode: "agent" } : {}), timeoutSec: 120, lifecycleMode: "per_turn" } }); + check("explicit-per-turn-policy", configured.adapterConfig?.acpxPermissionMode === "approve-reads" && configured.adapterConfig?.lifecycleMode === "per_turn" && (provider !== "cursor" || configured.adapterConfig?.acpxSessionMode === "agent"), "Agent mode and per-turn restrictive permission policy selected before startup"); + const project = await api.post(`/api/companies/${fixtures.company.id}/projects`, { name: `Native active Stop ${nonce}`, executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", allowIssueOverride: false, environmentId: fixtures.environment.id, workspaceStrategy: { type: "project_primary" } }, workspace: { name: "Primary", sourceType: "local_path", cwd: input.workspacePath, isPrimary: true } }); + if (!remote) await sample("before-request"); + await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title: execution.task.buildTitle(nonce), prompt: remote ? input.remoteBootstrap!.prompt(nonce) : prompt(), workMode: "standard", projectName: project.name }); + issue = (await pollUntil({ label: "browser-created active Stop task", deadlineAt: input.deadlineAt, load: async () => (await api.get(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(i => i.title === execution.task.buildTitle(nonce)), accept: Boolean }))!; + if (remote) { + await pollUntil({ label: "active Stop remote bootstrap", deadlineAt: input.deadlineAt, load, accept: state => state.run.status === "running" }); + const bound = await input.remoteBootstrap!.bindAndRelease({ issueId: issue.id, runId: runs[0]!.id, targets: [target], actionPrompt: async value => { + fixture = value; if (provider === "cursor") command = cursorDeniedCommand(join(value.remoteCwd, target)); + const prepared = await prepareCopilotRemoteAction({ fixture: value, companyId: fixtures.company.id, environmentId: fixtures.environment.id, runId: runs[0]!.id, target, prompt: prompt() }); + baseline = prepared.baseline; samples.push({ phase: "before-request", absent: true }); + await input.evidence("active-stop-before-request-remote.json", baseline); return prepared.prompt; + } }); + if (bound !== fixture) throw new Error("Remote active Stop publication identity changed"); + } + const label = "unanswered native permission"; + await pollUntil({ label, deadlineAt: input.deadlineAt, intervalMs: 200, load: async () => { + const state = await load(); if (["failed", "timed_out", "cancelled", "succeeded"].includes(state.run.status)) throw new Error(`Stopped waiting for ${label}: provider ended without pending callback`); return state; + }, accept: state => { observeActiveStopPending({ ...state, scope: scope(), cancellationRequestId: randomUUID(), bootstrap: bootstrap() }); return true; } }); + await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`); + const card = page.getByTestId("task-chat-runtime-request").filter({ visible: true }); + await expect(card).toHaveCount(1); await expect(card.getByRole("button", { name: "Deny", exact: true })).toBeEnabled(); + await input.capture("pending-permission", "Native permission left unanswered before Stop", "pending-permission.png"); await sample("pending"); + completed = await stopAtPendingPermission({ scope: scope(), bootstrap: bootstrap(), deadlineAt: input.deadlineAt, load, + retain: proof => input.evidence("native-active-stop-pending.json", proof), stop: (runId, cancellationRequestId) => api.post(`/api/heartbeat-runs/${runId}/cancel`, { cancellationRequestId }) }); + check("pending-callback-cancelled", true, "Unanswered native permission closed through a cancelled provider turn and the exact caller-owned Stop acknowledgement"); + await input.evidence("native-active-stop-settlement.json", completed); + // Only after confirmed cancellation: test the real public stale-response fence. + const response = await api.request.post(`/api/heartbeat-runs/${runs[0]!.id}/runtime-requests/${encodeURIComponent(completed.pending.requestId)}/resolve`, { data: { turnId: completed.pending.turnId, requestKind: "permission_approval", resolution: { action: "accept" } } }); + check("stale-answer-refused", response.status() === 409, "Stopped permission rejects a later answer rather than replaying work"); + await load(); readActiveStopSettlement({ ...await load(), ...completed, bootstrap: bootstrap() }); + if (remote) { sealed = await fixture!.finish(); assertCopilotRemoteRetirement(sealed, baseline!); processes = sealed.processes; } + await sample("after-stop"); + await page.reload(); await expect(card.getByRole("button", { name: "Deny", exact: true })).toHaveCount(0); + check("one-unfinished-cancelled-run", runs.length === 1 && issue.status === "in_progress" && runs[0]!.status === "cancelled", "No automatic follow-up run or false task completion"); + await input.capture("final-state", "Stopped native permission is no longer answerable", "final-state.png"); + await input.evidence("api-state.json", { issue, run: runs[0], runs, checks, samples, runEvents: events, runEventsByRun: [{ runId: runs[0]!.id, events }], activeStop: completed }); + return { issue, runs, checks }; + } finally { + // Keep watcher/process observation live through the registered cleanup assertion. + await input.evidence("native-active-stop-checks.json", { issue, runs, checks, samples, completed }); + } +} diff --git a/tests/runner-e2e/native-active-stop-tasks.ts b/tests/runner-e2e/native-active-stop-tasks.ts new file mode 100644 index 00000000000..3446ef501de --- /dev/null +++ b/tests/runner-e2e/native-active-stop-tasks.ts @@ -0,0 +1,10 @@ +import type { RunnerTaskFixture } from "./types.js"; +export const nativeActiveStopTasks: readonly RunnerTaskFixture[] = [{ + id: "pending-permission-stop", label: "Stop while native permission is unanswered", groups: [], workMode: "standard", flow: "native_active_stop", + expectedRunCount: 1, attemptTimeoutMs: { local: 300_000, daytona: 300_000 }, turnTimeoutMs: 120_000, + expectedTerminalState: { issue: "in_progress", run: "cancelled" }, + buildTitle: nonce => `Native pending permission Stop ${nonce}`, + buildVisibleMarker: nonce => `NATIVE-ACTIVE-STOP-${nonce}`, + buildPrompt: () => "Attempt the supplied exact native operation once. Leave its native permission callback pending for the board operator. Do not answer, reject or cancel your own permission; do not retry, use another tool/path, call paperclip_finish, or claim completion. The operator will stop this unfinished task while permission is still unanswered.", + buildMatchers: () => [], // Dedicated flow requires pending API observation, caller-owned Stop and retirement. +}]; diff --git a/tests/runner-e2e/native-active-stop.test.ts b/tests/runner-e2e/native-active-stop.test.ts new file mode 100644 index 00000000000..a320ab95e55 --- /dev/null +++ b/tests/runner-e2e/native-active-stop.test.ts @@ -0,0 +1,169 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { assertActiveStopRetirement, observeActiveStopPending, readActiveStopSettlement, type ActiveStopProvider } from "./native-active-stop-evidence.js"; +import { stopAtPendingPermission } from "./native-active-stop-flow.js"; +import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; +import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; + +const cancellationRequestId = "11111111-2222-4333-8444-555555555555"; +type Row = Record; +function fixture(provider: ActiveStopProvider = "copilot") { + const scope = { provider, companyId: "company", issueId: "issue", runId: "run", target: "target.txt", ...(provider === "cursor" ? { commandSha256: `sha256:${"a".repeat(64)}` } : {}) }; + const row = (seq: number, eventType: string, payload: Row): Row => ({ companyId: "company", runId: "run", seq, eventType, protocolSchemaVersion: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", eventType, runId: "run", turnId: "turn", + normalizedSessionId: "normalized-session", sourceInstanceId: "source", sourceSeq: seq, sourceEventId: `source:run:${seq}`, emittedAt: "2026-01-01T00:00:00Z", payload } } }); + const notice = (seq: number, stage: string, fields: Row) => row(seq, "provider.notice.recorded", { + schema: "paperclip.provider.notice.v1", scope: "turn", category: `${provider}_tool_evidence_v1`, + provenance: { sessionId: "native-session", turnId: "turn", eventType: stage, method: stage === "tool" ? "session/update" : "session/request_permission" }, + details: Object.entries({ stage, toolCallId: "tool", ...fields }).map(([name, value]) => ({ name, value: String(value) })), + }); + const operation = provider === "cursor" ? { operation: "execute", commandSha256: scope.commandSha256 } : { operation: "edit", target: scope.target }; + const events: Row[] = [ + notice(1, "tool", { status: "pending", ...operation }), + row(2, "tool.execution.started", { schema: "paperclip.tool.execution.v1", executionId: "tool", transport: "builtin", status: "running", ...operation }), + notice(3, "permission_requested", { requestId: "request", declineOffered: true, ...operation }), + row(4, "runtime_request.created", { request: { schema: "paperclip.runtime_request.v2", requestKind: "permission_approval", type: "permission", status: "pending", requestId: "request", turnId: "turn", itemId: "tool", + details: { toolCallId: "tool" }, origin: { adapter: "acpx-runtime-sidecar", provider, method: "session/request_permission" } } }), + ]; + const run: Row = { id: "run", companyId: "company", nativeIssueId: "issue", runtimeMode: "native", status: "running", resultJson: {} }; + const issue = { id: "issue", companyId: "company", status: "in_progress" }; + const state = () => ({ events, run, issue }); + const pending = () => observeActiveStopPending({ ...state(), scope, cancellationRequestId }); + const settle = (requestId = cancellationRequestId) => { + events.push(row(5, "runtime_request.cancelled", { provider: "acpx", requestId: "request", turnId: "turn", requestKind: "permission_approval", requestType: "permission", itemId: "tool", reason: "explicit_cancellation", replayAllowed: false }), + row(6, "turn.cancelled", { provider: "acpx", providerTurnId: "turn", status: "cancelled", error: null })); + run.status = "cancelled"; run.resultJson = { startupCancellation: { cancellationRequestId: requestId }, nativeCancellation: { + schema: "paperclip.native-cancellation.v1", intentId: `native-cancellation:${requestId}`, companyId: "company", runId: "run", issueId: "issue", scope: "run", + dispatched: true, dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", + } }; + return run; + }; + return { scope, events, run, issue, row, notice, state, pending, settle }; +} +const frame = (row: Row) => row.payload.prpEvent; +const payload = (row: Row) => frame(row).payload; +function settled() { const f = fixture(); const pending = f.pending(); f.settle(); return { f, pending, read: () => readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() }) }; } + +describe("definitely active native permission Stop", () => { + it.each(["cursor", "copilot"] as const)("binds %s's unanswered callback to cancelled provider settlement and caller-owned Stop", provider => { + const f = fixture(provider), pending = f.pending(); f.settle(); + expect(readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() })).toMatchObject({ + schema: "paperclip.e2e.native-active-stop-settlement.v1", branch: "pending_permission_cancelled", normalCompletionAccepted: false, taskStillInProgress: true, replayAllowed: false, + pending: { normalizedSessionId: "normalized-session", nativeSessionId: "native-session", cancellationRequestId }, + }); + }); + it("admits only a fully attested remote bootstrap read completed before the pending operation", () => { + const f = fixture(), actionFile = `.paperclip-eval-action-${"a".repeat(36)}.txt`; + const readTargetSha256 = `sha256:${createHash("sha256").update(actionFile).digest("hex")}`; + for (const row of f.events) { row.seq += 4; frame(row).sourceSeq += 4; frame(row).sourceEventId = `source:run:${row.seq}`; } + f.events.unshift(f.notice(1, "tool", { toolCallId: "bootstrap", operation: "read", status: "pending", readTargetSha256 }), + f.row(2, "tool.execution.started", { schema: "paperclip.tool.execution.v1", executionId: "bootstrap", transport: "builtin", status: "running", operation: "read", target: actionFile }), + f.notice(3, "tool", { toolCallId: "bootstrap", operation: "read", status: "completed", readTargetSha256 }), + f.row(4, "tool.execution.completed", { schema: "paperclip.tool.execution.v1", executionId: "bootstrap", transport: "builtin", status: "completed", operation: "read", target: actionFile })); + const bootstrap = { actionFile, events: f.events }; + expect(() => observeActiveStopPending({ ...f.state(), scope: f.scope, cancellationRequestId, bootstrap })).not.toThrow(); + payload(f.events[2]!).details.find((d: Row) => d.name === "readTargetSha256").value = `sha256:${"0".repeat(64)}`; + expect(() => observeActiveStopPending({ ...f.state(), scope: f.scope, cancellationRequestId, bootstrap })).toThrow(/bootstrap/); + }); + it("does not compare remote wall clock or transaction timestamps to the operator clock", () => { + const f = fixture(); for (const row of f.events) { frame(row).emittedAt = "2099-01-01T00:00:00Z"; row.createdAt = "1900-01-01T00:00:00Z"; } + const pending = f.pending(); f.settle(); + expect(() => readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: (BigInt(pending.observedMonotonicNs) + 1n).toString() })).not.toThrow(); + }); + it.each([ + ["prior Stop claim", (f: ReturnType) => { f.run.resultJson.startupCancellation = { cancellationRequestId }; }], + ["answered callback", (f: ReturnType) => { f.events.push(f.notice(5, "permission_delivered", { requestId: "request", outcome: "cancel" })); }], + ["already settled provider", (f: ReturnType) => { f.events.push(f.row(5, "turn.completed", { status: "completed" })); }], + ["already closed request", (f: ReturnType) => { f.events.push(f.row(5, "runtime_request.resolved", {})); }], + ["failed operation", (f: ReturnType) => { f.events.push(f.row(5, "tool.execution.completed", { schema: "paperclip.tool.execution.v1", transport: "builtin", executionId: "tool", status: "failed" })); }], + ] as const)("refuses %s before dispatch", (_label, mutate) => { const f = fixture(); mutate(f); expect(f.pending).toThrow(); }); + it.each([ + ["completed terminal", ({ f }: ReturnType) => { f.events[5]!.eventType = frame(f.events[5]!).eventType = "turn.completed"; payload(f.events[5]!).status = "completed"; }], + ["interrupted terminal", ({ f }: ReturnType) => { f.events[5]!.eventType = frame(f.events[5]!).eventType = "turn.interrupted"; payload(f.events[5]!).status = "interrupted"; }], + ["failed terminal", ({ f }: ReturnType) => { payload(f.events[5]!).error = { code: "failure" }; }], + ["duplicate terminal", ({ f }: ReturnType) => { f.events.push(f.row(7, "turn.cancelled", payload(f.events[5]!))); }], + ["late same-tool replay", ({ f }: ReturnType) => { f.events.push(f.notice(7, "tool", { status: "in_progress" })); }], + ["wrong closed item", ({ f }: ReturnType) => { payload(f.events[4]!).itemId = "other"; }], + ["reordered source stream", ({ f }: ReturnType) => { frame(f.events[5]!).sourceSeq = 1; frame(f.events[5]!).sourceEventId = "source:run:1"; }], + ["missing terminal", ({ f }: ReturnType) => { f.events.pop(); }], + ["wrapper type mismatch", ({ f }: ReturnType) => { f.events[5]!.eventType = "turn.completed"; }], + ["foreign normalized session", ({ f }: ReturnType) => { frame(f.events[5]!).normalizedSessionId = "native-session"; }], + ["foreign source", ({ f }: ReturnType) => { frame(f.events[5]!).sourceInstanceId = "foreign"; }], + ["foreign turn", ({ f }: ReturnType) => { frame(f.events[5]!).turnId = "other-turn"; }], + ["foreign company", ({ f }: ReturnType) => { f.events[5]!.companyId = "other"; }], + ["foreign request", ({ f }: ReturnType) => { payload(f.events[4]!).requestId = "other"; }], + ["expired rather than cancelled", ({ f }: ReturnType) => { f.events[4]!.eventType = frame(f.events[4]!).eventType = "runtime_request.expired"; }], + ["provider death", ({ f }: ReturnType) => { payload(f.events[4]!).reason = "provider_process_lost"; }], + ["replay permitted", ({ f }: ReturnType) => { payload(f.events[4]!).replayAllowed = true; }], + ["tampered pending row", ({ f }: ReturnType) => { payload(f.events[3]!).request.itemId = "changed"; }], + ["missing pending receipt", (s: ReturnType) => { s.pending.requestRowSha256 = ""; }], + ["foreign intent", ({ f }: ReturnType) => { f.run.resultJson.nativeCancellation.intentId = "native-cancellation:other"; }], + ["foreign ack scope", ({ f }: ReturnType) => { f.run.resultJson.nativeCancellation.companyId = "other"; }], + ["undispatched cancellation", ({ f }: ReturnType) => { f.run.resultJson.nativeCancellation.dispatched = false; }], + ["same audit twice", ({ f }: ReturnType) => { f.run.resultJson.nativeCancellation.acknowledgementAuditId = "audit-intent"; }], + ["finished task", ({ f }: ReturnType) => { f.issue.status = "done"; }], + ["extra semantic operation", ({ f }: ReturnType) => { f.events.push(f.row(7, "tool.execution.started", { executionId: "other" })); }], + ["extra native operation", ({ f }: ReturnType) => { f.events.push(f.notice(7, "tool", { toolCallId: "other", status: "pending" })); }], + ["native session switched", ({ f }: ReturnType) => { payload(f.events[0]!).provenance.sessionId = "other"; }], + ["target operation completed", ({ f }: ReturnType) => { f.events.push(f.row(7, "tool.execution.completed", { schema: "paperclip.tool.execution.v1", transport: "builtin", executionId: "tool", status: "completed" })); }], + ] as const)("rejects %s", (_label, mutate) => { const s = settled(); mutate(s); expect(s.read).toThrow(); }); + it("rejects a Stop dispatch not causally after the pending observation", () => { + const { f, pending } = settled(); + expect(() => readActiveStopSettlement({ ...f.state(), pending, dispatchMonotonicNs: pending.observedMonotonicNs })).toThrow(/pre-dispatch/); + }); +}); + +describe("active Stop flow wiring", () => { + it("awaits retention and rechecks pending before issuing the single caller UUID request", async () => { + const f = fixture(); const order: string[] = []; let retainedId = ""; + const stop = vi.fn(async (runId: string, id: string) => { expect(runId).toBe("run"); expect(id).toBe(retainedId); order.push("stop"); return f.settle(id); }); + const result = await stopAtPendingPermission({ scope: f.scope, deadlineAt: Date.now() + 1000, + load: async () => { order.push("load"); return f.state(); }, + retain: async receipt => { order.push("retain-start"); await Promise.resolve(); retainedId = receipt.cancellationRequestId; order.push("retain-done"); }, stop }); + expect(order).toEqual(["load", "retain-start", "retain-done", "load", "stop", "load"]); + expect(stop).toHaveBeenCalledTimes(1); expect(result.settlement.branch).toBe("pending_permission_cancelled"); + }); + it.each(["retention failed", "competing Stop", "answered after capture"])("never sends Stop when %s", async mode => { + const f = fixture(), stop = vi.fn(); + await expect(stopAtPendingPermission({ scope: f.scope, deadlineAt: Date.now() + 1000, load: async () => f.state(), stop, + retain: async () => { if (mode === "retention failed") throw Error("disk failure"); if (mode === "competing Stop") f.run.resultJson.startupCancellation = { cancellationRequestId: "foreign" }; else f.events.push(f.row(5, "runtime_request.resolved", {})); }, + })).rejects.toThrow(); expect(stop).not.toHaveBeenCalled(); + }); + it("fails promptly on a definitive provider failure before calling a throwing evidence reader", async () => { + const f = fixture(); const started = Date.now(); + await expect(stopAtPendingPermission({ scope: f.scope, deadlineAt: started + 10000, load: async () => f.state(), retain: async () => {}, stop: async (_run, id) => { + f.settle(id); f.run.status = "failed"; f.events.length = 0; return f.run; + } })).rejects.toThrow(/Stopped waiting.*unexpected run terminal/); + expect(Date.now() - started).toBeLessThan(500); + }); +}); + +describe("explicit active Stop discovery", () => { + it("adds exactly four versioned cells without enabling them in --all", () => { + const suite = runnerSuites.find(s => s.id === "native-active-stop")!; + const cells = runnerMatrix.filter(e => e.suite === suite); + expect(cells).toHaveLength(4); expect(suite.manualOnly).toBe(true); + expect(cells.map(e => `${e.profile.qualificationCandidate}/${e.environment.id}`).sort()).toEqual(["copilot/daytona", "copilot/local", "cursor/daytona", "cursor/local"]); + expect(cells.every(e => e.task.expectedRunCount === 1 && e.task.flow === "native_active_stop" && e.task.expectedTerminalState?.run === "cancelled")).toBe(true); + expect(suite.definitionMetadata).toMatchObject({ version: 1, normalCompletionAccepted: false, providerDeath: "not-covered" }); + expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, normalCompletionAccepted: true } })); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); + }); +}); + + +describe("active Stop cleanup remains mandatory", () => { + const proof = () => ({ completed: true, identityChanged: false, processes: { captured: true, live: [] as number[] }, + watcher: { complete: true, targetMutationCount: 0 }, samples: ["before-request", "pending", "after-stop", "after-cleanup"].map(phase => ({ phase, absent: true })) }); + it("requires all causal samples plus the continuous watcher and retired owned identities", () => expect(() => assertActiveStopRetirement(proof())).not.toThrow()); + it.each([ + ["live child", (p: ReturnType) => { p.processes.live.push(42); }], + ["unobserved process", (p: ReturnType) => { p.processes.captured = false; }], + ["replaced root", (p: ReturnType) => { p.identityChanged = true; }], + ["incomplete watch", (p: ReturnType) => { p.watcher.complete = false; }], + ["transient write", (p: ReturnType) => { p.watcher.targetMutationCount = 1; }], + ["written target", (p: ReturnType) => { p.samples[2]!.absent = false; }], + ["missing final sample", (p: ReturnType) => { p.samples.pop(); }], + ["missing settlement", (p: ReturnType) => { p.completed = false; }], + ] as const)("rejects %s even after an apparent cancelled run", (_label, mutate) => { const p = proof(); mutate(p); expect(() => assertActiveStopRetirement(p)).toThrow(); }); +}); diff --git a/tests/runner-e2e/prerequisites.ts b/tests/runner-e2e/prerequisites.ts index cec9d419985..1fdc8b5cd3e 100644 --- a/tests/runner-e2e/prerequisites.ts +++ b/tests/runner-e2e/prerequisites.ts @@ -18,7 +18,7 @@ export function assertRemoteNativeEvidencePrerequisites( environment: NodeJS.ProcessEnv, ): void { if (!executions.some(execution => execution.environment.id === "daytona" - && ["cursor_native", "pi_native", "copilot_protection"].includes(execution.task.flow))) return; + && ["cursor_native", "pi_native", "copilot_protection", "native_active_stop"].includes(execution.task.flow))) return; const names = ["PAPERCLIP_E2E_DAYTONA_NODE_SHA256", "PAPERCLIP_E2E_DAYTONA_RUNNERD_SHA256"]; const invalid = names.filter(name => !/^sha256:[a-f0-9]{64}$/u.test(environment[name] ?? "")); if (invalid.length) throw new Error(`Native Daytona evidence requires exact image executable digests: ${invalid.join(", ")}`); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 96cb90167a1..cfe715cdb28 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { runNativeActiveStopFlow } from "./native-active-stop-flow.js"; import { runCursorNativeFlow } from "./cursor-native-flow.js"; import { createRemoteNativeBootstrap, createRemoteFixtureClient } from "./remote-native-bootstrap.js"; import { runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js"; @@ -554,7 +555,7 @@ for (const execution of executions) { const credentials = credentialValues(); const secrets = normalizedSecrets(Object.values(credentials)); const api = new RunnerApi(request); - const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection", "cursor_native"].includes(execution.task.flow); + const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection", "cursor_native", "native_active_stop"].includes(execution.task.flow); const consoleDiagnostics: Array> = []; const networkDiagnostics: Array> = []; const pageLifecycleDiagnostics: Array> = []; @@ -846,7 +847,7 @@ for (const execution of executions) { }); const remoteBootstrap = execution.environment.id === "daytona" - && ["cursor_native", "pi_native", "copilot_protection"].includes(execution.task.flow) + && ["cursor_native", "pi_native", "copilot_protection", "native_active_stop"].includes(execution.task.flow) ? createRemoteNativeBootstrap({ api, daytona: await createRemoteFixtureClient(credentials.DAYTONA_API_KEY ?? ""), companyId: fixtures.company.id, environmentId: fixtures.environment.id, agentId: fixtures.agent.id, @@ -913,6 +914,15 @@ for (const execution of executions) { evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), }); issue = accounting.issue as IssueRecord; selectedRuns = accounting.runs as RunRecord[]; + } else if (execution.task.flow === "native_active_stop") { + const story = await runNativeActiveStopFlow({ + page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs, + observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; }, + capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), + remoteBootstrap, registerCleanupAssertion, registerBeforeEnvironmentTeardownAssertion, + }); + issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[]; + matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `nativeActiveStop.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })); } else if (execution.task.flow === "cursor_native") { const story = await runCursorNativeFlow({ page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 339840e8d3c..0265ffaded2 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -30,6 +30,7 @@ export type RunnerTaskFlow = | "warm_three_turn" | "instruction_persistence" | "pi_native" + | "native_active_stop" | "copilot_protection" | "cursor_native";