diff --git a/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts index 31749f638f1..1c2f877ec76 100644 --- a/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts +++ b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts @@ -41,6 +41,13 @@ describe("native heartbeat cancellation authority", () => { ); }); + it("passes the caller identity into the audited native boundary", async () => { + const cancel = vi.fn(async () => ({ decision: {}, auditId: "audit" })); + const db = {} as Db, cancellationRequestId = "11111111-1111-4111-8111-111111111111"; + await cancelHeartbeatNativeRun({ db, runId: "run", reason: "Stop", runtimeMode: "native", cancellationRequestId, cancel }); + expect(cancel).toHaveBeenCalledWith("run", "Stop", { db, scope: "run", cancellationRequestId }); + }); + it("fails closed when a native cancellation lacks its decision audit", async () => { const cancel = vi.fn(async () => ({ decision: null, diff --git a/server/src/__tests__/native-cancellation-request.integration.test.ts b/server/src/__tests__/native-cancellation-request.integration.test.ts new file mode 100644 index 00000000000..15d4d554471 --- /dev/null +++ b/server/src/__tests__/native-cancellation-request.integration.test.ts @@ -0,0 +1,54 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { cancellationIntentId, claimCancellationRequest, startupCancellationFence } from "../services/native-runtime/native-cancellation-request.js"; + +describe("atomic caller cancellation request ownership", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("caller-stop-"); db = createDb(database.connectionString); }, 90_000); + afterAll(async () => { await database?.cleanup(); }); + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Stop claims", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Stop target" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "running", runtimeMode: "native" }); + return { companyId, runId }; + } + const defaultFence = (runId: string) => db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('startupCancellation', ${startupCancellationFence(new Date().toISOString())})` }).where(eq(heartbeatRuns.id, runId)); + it("allows exactly one of two concurrent identities and idempotent same-ID retries", async () => { + const { companyId, runId } = await fixture(), a = randomUUID(), b = randomUUID(); + const results = await Promise.allSettled([claimCancellationRequest(db, runId, companyId, a, "board-user"), claimCancellationRequest(db, runId, companyId, b, "board-user")]); + expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1); + const winner = results[0].status === "fulfilled" ? a : b; + await expect(claimCancellationRequest(db, runId, companyId, winner, "other-board-user")).rejects.toMatchObject({ status: 409 }); + const copies = await Promise.all([claimCancellationRequest(db, runId, companyId, winner, "board-user"), claimCancellationRequest(db, runId, companyId, winner, "board-user")]); + expect(copies[0].resultJson).toEqual(copies[1].resultJson); + await expect(claimCancellationRequest(db, runId, randomUUID(), winner, "board-user")).rejects.toMatchObject({ status: 409 }); + }); + it("does not overwrite a caller claim when default Stop races after it", async () => { + const { companyId, runId } = await fixture(), id = randomUUID(); + const claimed = await claimCancellationRequest(db, runId, companyId, id, "board-user"); + await Promise.all([defaultFence(runId), claimCancellationRequest(db, runId, companyId, id, "board-user")]); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run.resultJson).toEqual(claimed.resultJson); + }); + it("rejects a default Stop that won first, without replacing its marker", async () => { + const { companyId, runId } = await fixture(); await defaultFence(runId); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await expect(claimCancellationRequest(db, runId, companyId, randomUUID(), "board-user")).rejects.toMatchObject({ status: 409 }); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(after.resultJson).toEqual(before.resultJson); + }); + it("accepts only the original acknowledged identity on a terminal run", async () => { + const { companyId, runId } = await fixture(), id = randomUUID(); + const run = await claimCancellationRequest(db, runId, companyId, id, "board-user"); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { ...run.resultJson, nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: cancellationIntentId(id), runId, companyId, scope: "run", dispatched: true, dispatchState: "acknowledged", intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() } } }).where(eq(heartbeatRuns.id, runId)); + await expect(claimCancellationRequest(db, runId, companyId, id, "board-user")).resolves.toMatchObject({ status: "cancelled" }); + await expect(claimCancellationRequest(db, runId, companyId, randomUUID(), "board-user")).rejects.toMatchObject({ status: 409 }); + await db.update(heartbeatRuns).set({ resultJson: { ...run.resultJson, nativeCancellation: { intentId: cancellationIntentId(id) } } }).where(eq(heartbeatRuns.id, runId)); + await expect(claimCancellationRequest(db, runId, companyId, id, "board-user")).rejects.toMatchObject({ status: 409 }); + }); +}); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 756d0c7c377..2471ca66d3c 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -1,3 +1,4 @@ +import { cancellationRequestId } from "../services/native-runtime/native-cancellation-request.js"; import { agentFileStore, agentFileTokenFromHash } from "../services/agent-file-store.js"; import { pipeline } from "node:stream/promises"; import { resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared"; @@ -6958,10 +6959,12 @@ export function agentRoutes( const runId = readHeartbeatRunId(req); const existing = await getAccessibleResource(req, res, heartbeat.getRun(runId), "Heartbeat run not found"); if (!existing) return; + const requestId = cancellationRequestId(req.body?.cancellationRequestId); // Stamp the cancellation as operator-initiated (this route is board-only). // Recovery reads this to stand down instead of classifying the cancelled // run as agent stranding and re-waking the agent the operator just stopped. const run = await heartbeat.cancelRun(runId, "Cancelled by a board operator", { + ...(requestId ? { cancellationRequestId: requestId, cancellationRequestedByUserId: req.actor.userId ?? null } : {}), resultJson: { cancelledByActorType: "user", cancelledByUserId: req.actor.userId ?? null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index fd341bae281..ba39981ae78 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { claimCancellationRequest, startupCancellationFence } from "./native-runtime/native-cancellation-request.js"; import { CHAT_COMPLETION_WAKE_REASON, prepareChatCompletionTurn, chatCompletionInstruction, isCompletedOnboardingHandoffWake } from "./chat-completion-delivery.js"; import { isAgentDirectoryCopy } from "./agent-directory-working-copies.js"; @@ -9358,10 +9359,11 @@ export async function cancelHeartbeatNativeRun(input: { runId: string; reason: string; runtimeMode: string | null; + cancellationRequestId?: string; cancel?: ( runId: string, reason: string, - options: { db: Db; scope: "run" }, + options: { db: Db; scope: "run"; cancellationRequestId?: string }, ) => Promise<{ decision: unknown | null; auditId: string | null }>; }) { if (input.runtimeMode !== "native") { @@ -9371,10 +9373,12 @@ export async function cancelHeartbeatNativeRun(input: { ? await input.cancel(input.runId, input.reason, { db: input.db, scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), }) : await cancelNativeSession(input.runId, input.reason, { db: input.db, scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), }); if (!cancellation.decision || !cancellation.auditId) { throw new Error("native_cancellation_outcome_not_audited"); @@ -29108,6 +29112,9 @@ export function heartbeatService( } type CancelRunOptions = { + /** Optional board request identity, atomically reserved for native Stop. */ + cancellationRequestId?: string; + cancellationRequestedByUserId?: string | null; errorCode?: string; resultJson?: Record; eventMessage?: string; @@ -29136,6 +29143,9 @@ export function heartbeatService( ) { let run = await getRun(runId); if (!run) throw notFound("Heartbeat run not found"); + if (options.cancellationRequestId) { + run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); + } const pendingNativeRetry = run.runtimeMode === "native" && run.status === "failed" ? await db @@ -29181,20 +29191,14 @@ export function heartbeatService( // Established legacy processes must still be stopped if the database is // unavailable. Only native or not-yet-dispatched preparation needs this // additional durable fence before its existing cancellation path. - if (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)) { + if (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control))) { const [fenced] = await db.update(heartbeatRuns).set({ // Record handoff intent before native cancellation can finalize and release // the run. Only its audited stop acknowledgement suppresses recovery. resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true ? { reassignmentStopRequested: true } : {})}::jsonb || - jsonb_build_object('startupCancellation', jsonb_build_object( - 'requestedAt', ${new Date().toISOString()}::text, - 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.runtimeModeResolvedAt} is null - and ${heartbeatRuns.executionStage} = 'preparing' - and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) - ))`, + jsonb_build_object('startupCancellation', ${startupCancellationFence(new Date().toISOString())})`, }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], ))).returning(); @@ -29270,6 +29274,7 @@ export function heartbeatService( runId: run.id, reason, runtimeMode: run.runtimeMode, + ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), }); if (running) { await terminateHeartbeatRunProcess({ diff --git a/server/src/services/native-runtime/native-cancellation-request.test.ts b/server/src/services/native-runtime/native-cancellation-request.test.ts new file mode 100644 index 00000000000..352f1f80ea3 --- /dev/null +++ b/server/src/services/native-runtime/native-cancellation-request.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { assertCancellationRequest, cancellationIntentId, cancellationRequestId } from "./native-cancellation-request.js"; +const id = "11111111-1111-4111-8111-111111111111"; +describe("caller-bound native cancellation", () => { + it.each([null, "", " x", {}, [], 1, `${id} `, "not-a-uuid"])("rejects malformed request %j", value => { + expect(() => cancellationRequestId(value)).toThrow("Invalid cancellationRequestId"); + }); + it("preserves the absent option and normalizes a valid UUID", () => { + expect(cancellationRequestId(undefined)).toBeUndefined(); + expect(cancellationRequestId("AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA")).toBe("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + }); + it.each([{ startupCancellation: {} }, { startupCancellation: { requestedAt: "earlier" } }, + { startupCancellation: { cancellationRequestId: "other" } }, { nativeCancellation: { intentId: "other" } }])("rejects prior or racing intent %j", result => { + expect(() => assertCancellationRequest(result, id, true)).toThrow("earlier Stop"); + }); + it("requires a reservation at the native locked dispatch gate", () => { + expect(() => assertCancellationRequest({}, id)).toThrow("earlier Stop"); + expect(() => assertCancellationRequest({}, id, true)).not.toThrow(); + const startupCancellation = { cancellationRequestId: id }; + expect(() => assertCancellationRequest({ startupCancellation }, id)).not.toThrow(); + expect(() => assertCancellationRequest({ startupCancellation, nativeCancellation: { intentId: cancellationIntentId(id) } }, id)).not.toThrow(); + }); +}); diff --git a/server/src/services/native-runtime/native-cancellation-request.ts b/server/src/services/native-runtime/native-cancellation-request.ts new file mode 100644 index 00000000000..ecdf8d4ec59 --- /dev/null +++ b/server/src/services/native-runtime/native-cancellation-request.ts @@ -0,0 +1,71 @@ +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { badRequest, conflict } from "../../errors.js"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const record = (value: unknown): Record => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +export function cancellationRequestId(value: unknown): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string" || !UUID.test(value)) throw badRequest("Invalid cancellationRequestId"); + return value.toLowerCase(); +} +export const cancellationIntentId = (requestId: string) => `native-cancellation:${requestId}`; + +/** Run only while holding this company's run row lock. An earlier uncorrelated + * startup fence is also a conflicting Stop, even before native intent exists. */ +export function assertCancellationRequest(result: unknown, requestId: string, allowUnclaimed = false) { + const value = record(result), startup = record(value.startupCancellation), native = record(value.nativeCancellation); + if ((Object.hasOwn(value, "startupCancellation") && startup.cancellationRequestId !== requestId) + || (Object.hasOwn(value, "nativeCancellation") && native.intentId !== cancellationIntentId(requestId)) + || (!allowUnclaimed && startup.cancellationRequestId !== requestId)) { + throw conflict("Cancellation request conflicts with an earlier Stop"); + } +} + +/** Reserve the caller's identity before handoff or dispatch. No schema change: + * the existing startup fence and native intent carry the correlation. */ +export async function claimCancellationRequest(db: Db, runId: string, companyId: string, requestId: string, userId: string | null) { + if (cancellationRequestId(requestId) !== requestId) throw badRequest("Invalid cancellationRequestId"); + if (userId !== null && (typeof userId !== "string" || !userId)) throw badRequest("Invalid cancellation request actor"); + return db.transaction(async tx => { + const run = await tx.select().from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))) + .for("update").limit(1).then(rows => rows[0]); + if (!run || run.runtimeMode !== "native") throw conflict("Correlated Stop requires a native run"); + assertCancellationRequest(run.resultJson, requestId, true); + const result = record(run.resultJson); + if (record(result.startupCancellation).cancellationRequestId === requestId) { + const actor = record(record(result.startupCancellation).requestedBy); + if (actor.type !== "board" || actor.userId !== userId) throw conflict("Cancellation request belongs to another actor"); + if (!["queued", "running", "scheduled_retry"].includes(run.status)) { + const native = record(result.nativeCancellation); + if (run.status !== "cancelled" || native.schema !== "paperclip.native-cancellation.v1" + || native.intentId !== cancellationIntentId(requestId) || native.runId !== runId || native.companyId !== companyId + || native.scope !== "run" || native.dispatchState !== "acknowledged" || typeof native.dispatched !== "boolean" + || typeof native.intentAuditId !== "string" || !native.intentAuditId + || typeof native.acknowledgementAuditId !== "string" || !native.acknowledgementAuditId + || native.intentAuditId === native.acknowledgementAuditId) throw conflict("Correlated Stop run is already terminal without acknowledgement"); + } + return run; + } + if (!["queued", "running", "scheduled_retry"].includes(run.status)) throw conflict("Correlated Stop run is already terminal"); + const [claimed] = await tx.update(heartbeatRuns).set({ resultJson: { ...result, + startupCancellation: { requestedAt: new Date().toISOString(), beforeNativeSelection: false, cancellationRequestId: requestId, requestedBy: { type: "board", userId } }, + } }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).returning(); + if (!claimed) throw conflict("Correlated Stop run changed"); + return claimed; + }); +} + +/** Used by the default Stop path too: never erase a correlated reservation. */ +export function startupCancellationFence(requestedAt: string) { + return sql`CASE + WHEN ${heartbeatRuns.resultJson}->'startupCancellation'->>'cancellationRequestId' IS NOT NULL + THEN ${heartbeatRuns.resultJson}->'startupCancellation' + ELSE jsonb_build_object('requestedAt', ${requestedAt}::text, + 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' + and ${heartbeatRuns.runtimeModeResolvedAt} is null + and ${heartbeatRuns.executionStage} = 'preparing' + and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)) + END`; +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b6916345f3d..2c543de667d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4919,6 +4919,7 @@ function cancellationDb(options?: { } | null; failResultJsonUpdateAt?: number; ownershipHeld?: boolean; + resultJson?: Record; }) { const initialRun = { id: execution.binding.runId, @@ -4938,6 +4939,7 @@ function cancellationDb(options?: { }; let currentResultJson: Record = { durableReceipt: { operationId: "operation-1" }, + ...options?.resultJson, }; const issue = { status: "in_progress", @@ -5505,6 +5507,34 @@ describe("native session cancellation", () => { expect(state.publishActivity).toHaveBeenCalledTimes(2); }); + it("uses the reserved caller UUID and permits same-intent and default retries", async () => { + const id = "11111111-1111-4111-8111-111111111111"; + const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } }); + await cancelNativeSession(execution.binding.runId, "operator Stop", { db: persistence.db, scope: "run", cancellationRequestId: id }); + expect(persistence.getResultJson().nativeCancellation).toMatchObject({ intentId: `native-cancellation:${id}` }); + const writes = persistence.getResultJsonUpdateCount(); + await cancelNativeSession(execution.binding.runId, "same request", { db: persistence.db, scope: "run", cancellationRequestId: id }); + await cancelNativeSession(execution.binding.runId, "default retry", { db: persistence.db, scope: "run" }); + expect(persistence.getResultJsonUpdateCount()).toBe(writes); + await expect(cancelNativeSession(execution.binding.runId, "foreign caller", { + db: persistence.db, scope: "run", cancellationRequestId: "22222222-2222-4222-8222-222222222222", + })).rejects.toMatchObject({ status: 409 }); + expect(persistence.getResultJsonUpdateCount()).toBe(writes); + }); + + it.each([{}, { startupCancellation: { requestedAt: "prior" } }, + { startupCancellation: { cancellationRequestId: "other" } }, + { startupCancellation: { cancellationRequestId: "11111111-1111-4111-8111-111111111111" }, nativeCancellation: { intentId: "foreign" } }, + ])("rejects missing or raced caller ownership under the intent lock: %j", async resultJson => { + const persistence = cancellationDb({ resultJson }); + await expect(cancelNativeSession(execution.binding.runId, "operator Stop", { + db: persistence.db, scope: "run", cancellationRequestId: "11111111-1111-4111-8111-111111111111", + })).rejects.toMatchObject({ status: 409 }); + expect(persistence.getForUpdateCount()).toBe(1); + expect(persistence.updates).toEqual([]); + expect(state.cancel).not.toHaveBeenCalled(); + }); + it("recovers a post-dispatch persistence failure without cancelling the provider twice", async () => { const persistence = cancellationDb({ failResultJsonUpdateAt: 2 }); const running = executePaperclipNativeSession({ diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e8ffeb681bc..6ce8fd47a2c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { assertCancellationRequest, cancellationIntentId as callerCancellationIntentId, cancellationRequestId } from "./native-cancellation-request.js"; import { readNativeCursorPlanWait } from "./native-cursor-plan-wait.js"; import { resolveAcpxQualification } from "./acpx-qualification.js"; import { readLocalAiCredentialFile } from "../local-ai-credential-file.js"; @@ -6527,6 +6528,7 @@ export function cancelNativeSession( db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean; + cancellationRequestId?: string; }, ): Promise<{ dispatched: boolean; @@ -6541,6 +6543,7 @@ export async function cancelNativeSession( db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean; + cancellationRequestId?: string; }, ): Promise< | boolean @@ -6683,6 +6686,11 @@ export async function cancelNativeSession( throw new Error("native_cancellation_coordinator_missing"); const resultJson = record(lockedRun.resultJson); + // A default Stop joins an already reserved caller intent; it cannot + // replace that intent while the originating request is still dispatching. + const requestedId = options.cancellationRequestId + ?? cancellationRequestId(record(resultJson.startupCancellation).cancellationRequestId); + if (requestedId) assertCancellationRequest(resultJson, requestedId); const existing = record(resultJson.nativeCancellation); const existingIntentId = typeof existing.intentId === "string" && existing.intentId.length > 0 @@ -6723,7 +6731,9 @@ export async function cancelNativeSession( }; } - const intentId = `native-cancellation:${randomUUID()}`; + const intentId = requestedId + ? callerCancellationIntentId(requestedId) + : `native-cancellation:${randomUUID()}`; const activity = await persistActivity(tx as unknown as Db, { companyId: cancellationContext.companyId, actorType: "system", diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 46dadd4049d..920387cd18e 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -286,7 +286,7 @@ provider timeout, 300s attempt budget). `native-permission-deny-write` denies on exact native edit through its browser card, waits for the delivered rejection and failed tool, then cancels through the public run API. Its expected outcome is a cancelled run and an unfinished task, not successful task completion. -The retained `paperclip.e2e.copilot-denial-settlement.v2` proof separates the exact +The retained `paperclip.e2e.copilot-denial-settlement.v3` proof separates the exact provider terminal from the audited controller Stop. It records either `provider_cancelled_or_interrupted` or `provider_completed_observed_before_stop`. The latter requires the exact terminal row to be returned by the operator API @@ -372,3 +372,22 @@ process-retirement receipt; their timestamps are not relabeled as host time. This denial case does not qualify Stop during a definitely pending native request. That active-turn cancellation boundary needs a separate live case. The attached async-command oracle is unchanged by this denial-only correction. + +### Correlated native Stop API + +The board-only `POST /api/heartbeat-runs/:runId/cancel` accepts an optional +`cancellationRequestId` UUID for native runs. Company access checks still apply. +The server reserves it under the run-row lock before dispatch, uses +`native-cancellation:` as the durable intent ID, and returns HTTP409 for +an earlier or different caller intent, an earlier uncorrelated Stop, or a +terminal run without that same reserved intent. Malformed UUIDs return HTTP400. +Repeating the same UUID from the same board actor is idempotent. A different +actor receives HTTP409; local trusted board uses an explicit null user ID. Default clients may omit the field; they +preserve and join an existing reserved intent rather than overwrite it. + +The denial fixture generates its UUID before observation, retains it in +`paperclip.e2e.copilot-pre-stop-observation.v2`, and requires the same intent in +the response and final `paperclip.e2e.copilot-denial-settlement.v3` receipt. +It refuses a non-running controller or existing Stop marker before dispatch. +The completed-provider branch still requires a running controller that this +request can stop; it does not accept a no-op Stop of an already terminal run. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index bb9a80b058d..1830e4b5587 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1408,10 +1408,23 @@ This denial case does not qualify Stop during a definitely pending native reques That active-turn cancellation boundary needs a separate live case. The attached async-command oracle is unchanged by this denial-only correction. -Copilot denial settlement v2 (suite definition v4) requires a retained pre-Stop API +Copilot denial settlement v3 (suite definition v5) requires a retained pre-Stop API observation. The fixture awaits that artifact write before dispatching Stop and matches exact source identities and row hashes against final evidence. It waits at most 2s for natural settlement within the existing deadline. Normal completion first observed afterward cannot pass. A row's createdAt is transaction-start time, not proof of commit order. Older failed artifacts lack this observation and cannot establish a completed-before-Stop causal boundary; they remain failed. + +The Copilot denial Stop cell requires the controller run to remain `running` +until its Stop request. A provider `turn.completed` can satisfy its completed +branch while that controller run remains active. An already `succeeded` or +`cancelled` controller run cannot satisfy this Stop-specific cell. Such a result +is not evidence that the provider bypassed denial; natural completion without +an active controller Stop needs separate coverage. + +Suite version 5 binds a fresh `cancellationRequestId` UUID in the retained +pre-Stop observation to the public cancel request and its exact durable native +intent. It rejects prior startup/Stop markers before dispatch and a competing +request that wins after the last read. The server atomically reserves the caller +identity; the fixture does not infer ownership from timestamps or HTTP success. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 9f42eb8c767..207b7565580 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1051,7 +1051,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 4, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v2", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 5, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-protection-cases.test.ts b/tests/runner-e2e/copilot-protection-cases.test.ts index 3f8c647ffde..d2768e649c4 100644 --- a/tests/runner-e2e/copilot-protection-cases.test.ts +++ b/tests/runner-e2e/copilot-protection-cases.test.ts @@ -7,11 +7,11 @@ const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succ const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 }; function denied(): CopilotDeniedWriteEvidence { return { - expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v2", ...identity, requestId: "permission-0", + expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v3", ...identity, requestId: "permission-0", branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true, - preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v1" as const, ...identity, requestId: "permission-0", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10" }, stopDispatchMonotonicNs: "20", + preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v2" as const, ...identity, requestId: "permission-0", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }, stopDispatchMonotonicNs: "20", providerTerminal: { rowSha256: `sha256:${"b".repeat(64)}`, failedToolRowSha256: `sha256:${"a".repeat(64)}`, eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolRowCreatedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, rowCreatedAtMs: 51 }, - runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", + runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "native-cancellation:11111111-1111-4111-8111-111111111111", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] }, decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" }, deliveredDecision: { ...identity, observedAtMs: 25, requestId: "permission-0", outcome: "reject_once" }, diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 3afb94fcbab..e93f21f1f38 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -196,7 +196,7 @@ export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRem } export interface CopilotDenialSettlement { - schema: "paperclip.e2e.copilot-denial-settlement.v2"; + schema: "paperclip.e2e.copilot-denial-settlement.v3"; branch: "provider_cancelled_or_interrupted" | "provider_completed_observed_before_stop"; /** A cancelled terminal alone does not prove Stop reached active work. */ providerCancellationTerminalObserved: boolean; @@ -222,7 +222,7 @@ const settlementTime = (v: unknown): v is number => Number.isSafeInteger(v) && ( const settlementDate = (v: unknown): number => typeof v === "string" && /^\d{4}-\d{2}-\d{2}T.*Z$/u.test(v) ? Date.parse(v) : NaN; export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolean { - if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v2") return false; + if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v3") return false; const t = s.providerTerminal, c = s.runStop; if (!t || !c || ![s.runId, s.sessionId, s.turnId, s.toolCallId, s.requestId, t.normalizedSessionId, t.sourceInstanceId, c.companyId, c.issueId, c.intentId, c.intentAuditId, c.acknowledgementAuditId].every(settlementId) @@ -233,8 +233,10 @@ export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolea || !Number.isSafeInteger(t.failedToolSourceSeq) || t.failedToolSourceSeq <= 0 || !Number.isSafeInteger(t.sourceSeq) || t.sourceSeq <= t.failedToolSourceSeq || c.requestedAtMs > c.recordedAtMs || c.recordedAtMs > c.acknowledgedAtMs || c.acknowledgedAtMs > c.finishedAtMs) return false; const before = s.preStop; - if (!before || before.schema !== "paperclip.e2e.copilot-pre-stop-observation.v1" + if (!before || before.schema !== "paperclip.e2e.copilot-pre-stop-observation.v2" || !["runId", "sessionId", "turnId", "toolCallId", "requestId"].every(k => before[k as keyof CopilotPreStopObservation] === s[k as keyof CopilotDenialSettlement]) + || typeof before.cancellationRequestId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u.test(before.cancellationRequestId) + || c.intentId !== `native-cancellation:${before.cancellationRequestId}` || before.companyId !== c.companyId || before.normalizedSessionId !== t.normalizedSessionId || before.sourceInstanceId !== t.sourceInstanceId || before.failedToolSourceSeq !== t.failedToolSourceSeq || before.failedToolRowSha256 !== t.failedToolRowSha256 || ![t.rowSha256, t.failedToolRowSha256].every(v => /^sha256:[a-f0-9]{64}$/u.test(v)) @@ -292,13 +294,15 @@ export function readCopilotDeniedEdit(input: { events: readonly unknown[]; reque } export interface CopilotPreStopObservation { - schema: "paperclip.e2e.copilot-pre-stop-observation.v1"; + schema: "paperclip.e2e.copilot-pre-stop-observation.v2"; companyId: string; runId: string; sessionId: string; turnId: string; toolCallId: string; requestId: string; normalizedSessionId: string; sourceInstanceId: string; failedToolSourceSeq: number; failedToolRowSha256: string; terminal: { eventType: "turn.completed" | "turn.cancelled" | "turn.interrupted"; sourceSeq: number; rowSha256: string } | null; /** Fixture-process monotonic clock, never provider or database wall time. */ apiReadCompletedMonotonicNs: string; + /** Fresh caller UUID reserved atomically by the board cancel API. */ + cancellationRequestId: string; } const denialRowSha = (row: unknown) => `sha256:${createHash("sha256").update(canonicalJson(row)).digest("hex")}`; function readDeniedTerminal(proof: ReturnType) { @@ -317,14 +321,14 @@ function readDeniedTerminal(proof: ReturnType) { } /** Call only on rows returned by the current operator API read. Retain this * receipt before dispatching Stop; historical rows cannot recreate that fact. */ -export function observeCopilotPreStop(input: { events: readonly unknown[]; request: CopilotToolNotice; companyId: string }): CopilotPreStopObservation { +export function observeCopilotPreStop(input: { events: readonly unknown[]; request: CopilotToolNotice; companyId: string; cancellationRequestId: string }): CopilotPreStopObservation { const proof = readCopilotDeniedEdit(input), observed = readDeniedTerminal(proof), r = input.request; - return { schema: "paperclip.e2e.copilot-pre-stop-observation.v1", companyId: input.companyId, + return { schema: "paperclip.e2e.copilot-pre-stop-observation.v2", companyId: input.companyId, runId: r.runId, sessionId: r.sessionId, turnId: r.turnId, toolCallId: r.toolCallId, requestId: r.requestId!, normalizedSessionId: proof.origin.normalizedSessionId, sourceInstanceId: proof.origin.sourceInstanceId, failedToolSourceSeq: proof.tool.sourceSeq, failedToolRowSha256: denialRowSha(proof.toolRows[0]), terminal: observed ? { eventType: observed.terminal.eventType, sourceSeq: observed.terminal.sourceSeq, rowSha256: denialRowSha(observed.row) } : null, - apiReadCompletedMonotonicNs: process.hrtime.bigint().toString() }; + apiReadCompletedMonotonicNs: process.hrtime.bigint().toString(), cancellationRequestId: input.cancellationRequestId }; } /** Denial closes one permission, not necessarily the provider prompt. Match the @@ -340,6 +344,8 @@ export function readCopilotDenialSettlement(input: { if (events.length > 20_000 || run.id !== request.runId || issue.id !== run.nativeIssueId || !settlementId(issue.companyId) || run.companyId !== issue.companyId || issue.status !== "in_progress" || run.status !== "cancelled" || stop.schema !== "paperclip.native-cancellation.v1" || stop.runId !== run.id || stop.companyId !== issue.companyId || stop.issueId !== issue.id + || stop.intentId !== `native-cancellation:${input.preStop?.cancellationRequestId}` + || settlementRecord(settlementRecord(run.resultJson).startupCancellation).cancellationRequestId !== input.preStop?.cancellationRequestId || stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only" || !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid(); const { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed } = readCopilotDeniedEdit({ events, request, companyId: issue.companyId }); @@ -347,7 +353,7 @@ export function readCopilotDenialSettlement(input: { if (!observed) throw invalid(); const { row, terminal } = observed; const result: CopilotDenialSettlement = { - schema: "paperclip.e2e.copilot-denial-settlement.v2", + schema: "paperclip.e2e.copilot-denial-settlement.v3", branch: row.eventType === "turn.completed" ? "provider_completed_observed_before_stop" : "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: row.eventType !== "turn.completed", runId: run.id, sessionId: request.sessionId, turnId: request.turnId, toolCallId: request.toolCallId, requestId: request.requestId!, diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index b13b2647399..d6d40158f61 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,8 +26,8 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 4, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", - denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v2", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 5, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); expect(cells.every(c => c.profile.qualificationCandidate === "copilot" && c.task.expectedRunCount === 1)).toBe(true); @@ -41,11 +41,11 @@ describe("Copilot Product protection integration", () => { request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] }, decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" }, deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! }, - toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v2" as const, ...copilotOrigin(request), requestId: "permission", + toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v3" as const, ...copilotOrigin(request), requestId: "permission", branch: "provider_cancelled_or_interrupted" as const, providerCancellationTerminalObserved: true, - preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v1" as const, ...copilotOrigin(request), requestId: "permission", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10" }, stopDispatchMonotonicNs: "20", + preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v2" as const, ...copilotOrigin(request), requestId: "permission", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }, stopDispatchMonotonicNs: "20", providerTerminal: { rowSha256: `sha256:${"b".repeat(64)}`, failedToolRowSha256: `sha256:${"a".repeat(64)}`, eventType: "turn.cancelled" as const, normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolRowCreatedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, rowCreatedAtMs: 51 }, - runStop: { companyId: "company", issueId: "issue", scope: "run" as const, status: "cancelled" as const, issueStatus: "in_progress" as const, intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, + runStop: { companyId: "company", issueId: "issue", scope: "run" as const, status: "cancelled" as const, issueStatus: "in_progress" as const, intentId: "native-cancellation:11111111-1111-4111-8111-111111111111", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1, fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false, providerCursor: index === 0 ? null : { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: [0, 1, 4, 6, 6][index]! } })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } }; expect(request.target).toBe(target); diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 91fc0633e1e..eb2b8902463 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -1,4 +1,4 @@ -import { createHash, randomBytes } from "node:crypto"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; import { join } from "node:path"; import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; @@ -21,11 +21,23 @@ export async function settleCopilotDeniedRun(input: { retainPreStop(receipt: CopilotPreStopObservation): Promise; afterSettlement(): Promise; }) { + const cancellationRequestId = randomUUID(); + const assertBeforeStop = (state: Awaited>) => { + if (state.run.id !== input.request.runId || state.run.status !== "running" + || state.run.resultJson?.startupCancellation != null || state.run.resultJson?.nativeCancellation != null) { + throw new Error("Copilot denial observed an earlier Stop or non-running run"); + } + }; + let stopSent = false; const poll = (label: string, accept: (state: Awaited>) => boolean) => pollUntil({ label, deadlineAt: input.deadlineAt, load: async () => { const state = await input.load(); // pollUntil recognizes this definitive rejection before invoking readers. if (["failed", "timed_out"].includes(state.run.status)) throw new Error(`Stopped waiting for ${label}: Copilot provider run failed`); + if (!stopSent) { + try { assertBeforeStop(state); } + catch { throw new Error(`Stopped waiting for ${label}: Copilot denial observed an earlier Stop or non-running run`); } + } return state; }, accept, intervalMs: 200, }); @@ -40,7 +52,8 @@ export async function settleCopilotDeniedRun(input: { while (true) { const state = await input.load(); if (["failed", "timed_out"].includes(state.run.status)) throw new Error("Copilot provider run failed before Stop"); - preStop = observeCopilotPreStop({ events: state.events, request: input.request, companyId: state.issue.companyId }); + assertBeforeStop(state); + preStop = observeCopilotPreStop({ events: state.events, request: input.request, companyId: state.issue.companyId, cancellationRequestId }); if (preStop.terminal || Date.now() >= observeUntil) break; await new Promise(resolve => setTimeout(resolve, Math.min(200, observeUntil - Date.now()))); } @@ -49,8 +62,14 @@ export async function settleCopilotDeniedRun(input: { // this causal boundary, and a later replay must never manufacture it. await input.retainPreStop(preStop); if (Date.now() >= input.deadlineAt) throw new Error("Copilot denial deadline reached before Stop"); + assertBeforeStop(await input.load()); + if (Date.now() >= input.deadlineAt) throw new Error("Copilot denial deadline reached before Stop"); const stopDispatchMonotonicNs = process.hrtime.bigint().toString(); - await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`); + const stopped = await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`, { cancellationRequestId }); + if (stopped?.id !== input.request.runId || stopped?.resultJson?.nativeCancellation?.intentId !== `native-cancellation:${cancellationRequestId}`) { + throw new Error("Copilot denial Stop response has a foreign cancellation intent"); + } + stopSent = true; await poll("correlated provider settlement and retired run", state => { if (!state.retired) return false; readCopilotDenialSettlement({ ...state, request: input.request, preStop, stopDispatchMonotonicNs }); return true; diff --git a/tests/runner-e2e/copilot-protection-settlement.test.ts b/tests/runner-e2e/copilot-protection-settlement.test.ts index a920e91f993..3b4e910e2f9 100644 --- a/tests/runner-e2e/copilot-protection-settlement.test.ts +++ b/tests/runner-e2e/copilot-protection-settlement.test.ts @@ -25,17 +25,27 @@ function fixture(eventType = "turn.completed") { frame(121, eventType, { status: eventType.slice(5), error: null }, 900, 887), frame(112, "runtime_request.resolved", { requestId: "request", turnId: "turn", requestKind: "permission_approval", action: "decline" })]; const run: any = { id: "run", companyId: "company", nativeIssueId: "issue", status: "cancelled", finishedAt: date(910), - resultJson: { startupCancellation: { requestedAt: date(892) }, nativeCancellation: { + resultJson: { startupCancellation: { requestedAt: date(892), cancellationRequestId: "11111111-1111-4111-8111-111111111111" }, nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: "run", issueId: "issue", scope: "run", dispatched: true, - dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentId: "intent", + dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentId: "native-cancellation:11111111-1111-4111-8111-111111111111", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", recordedAt: date(901), acknowledgedAt: date(909), } } }; const request = readCopilotToolEvidence(events, "run")[0]!; - const preStop = observeCopilotPreStop({ events, request, companyId: "company" }); + const preStop = observeCopilotPreStop({ events, request, companyId: "company", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }); return { events, request, run, issue: { id: "issue", companyId: "company", status: "in_progress" }, preStop, stopDispatchMonotonicNs: process.hrtime.bigint().toString() }; } const terminal = (f: ReturnType) => f.events[4]!.payload.prpEvent; +const liveRun = (f: ReturnType) => ({ ...f.run, status: "running", resultJson: {} }); +function stopPost(f: ReturnType, callback = () => {}) { + return vi.fn(async (_path: string, body: { cancellationRequestId: string }) => { + callback(); + f.run.resultJson.startupCancellation.cancellationRequestId = body.cancellationRequestId; + f.run.resultJson.nativeCancellation.intentId = `native-cancellation:${body.cancellationRequestId}`; + return f.run; + }); +} + describe("Copilot denial provider settlement and separate audited run Stop", () => { it("accepts a synthetic pre-Stop API observation without claiming provider cancellation", () => { const f = fixture(); @@ -43,7 +53,7 @@ describe("Copilot denial provider settlement and separate audited run Stop", () providerTerminal: { sourceSeq: 121, failedToolSourceSeq: 115, rowCreatedAtMs: Date.parse(date(900)) }, runStop: { status: "cancelled" } }); // Native/provider clock skew is irrelevant to the operator causal boundary. terminal(f).emittedAt = date(999); - f.preStop = observeCopilotPreStop({ ...f, companyId: "company" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); + f.preStop = observeCopilotPreStop({ ...f, companyId: "company", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_observed_before_stop"); }); it.each(["turn.cancelled", "turn.interrupted"])("retains the distinct observed %s branch", type => { @@ -55,7 +65,7 @@ describe("Copilot denial provider settlement and separate audited run Stop", () }); it.each(["equal", "later"])("accepts observed pre-Stop completion with %s transaction time, without ordering it against ack", kind => { const f = fixture(); f.events[4]!.createdAt = date(kind === "equal" ? 909 : 910); - f.preStop = observeCopilotPreStop({ ...f, companyId: "company" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); + f.preStop = observeCopilotPreStop({ ...f, companyId: "company", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_observed_before_stop"); }); it("rejects late normal completion even when its transaction began before Stop ack", () => { @@ -134,14 +144,14 @@ describe("Copilot denial provider settlement and separate audited run Stop", () try { const f = fixture("turn.cancelled"); let loads = 0, dispatched = false, postLoads = 0; let release!: () => void; const retention = new Promise(resolve => { release = resolve; }); - const post = vi.fn(async () => { dispatched = true; }); + const post = stopPost(f, () => { dispatched = true; }); const retainPreStop = vi.fn(async (receipt) => { expect(receipt.terminal).toBeNull(); await retention; }); const afterDeniedEdit = vi.fn(async () => {}), afterSettlement = vi.fn(async () => {}); const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 5000, load: async () => { loads++; if (dispatched) postLoads++; const events = f.events.filter((_, i) => !(loads === 1 && i === 3) && !((!dispatched || postLoads === 1) && i === 4)); - return { ...f, events, run: dispatched ? f.run : { ...f.run, status: "running" }, retired: dispatched }; + return { ...f, events, run: dispatched ? f.run : liveRun(f), retired: dispatched }; }, afterDeniedEdit, retainPreStop, afterSettlement }); await vi.advanceTimersByTimeAsync(0); expect(post).not.toHaveBeenCalled(); expect(afterDeniedEdit).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(200); expect(afterDeniedEdit).toHaveBeenCalledOnce(); expect(post).not.toHaveBeenCalled(); @@ -154,10 +164,10 @@ describe("Copilot denial provider settlement and separate audited run Stop", () vi.useFakeTimers(); let mono = 100n; vi.spyOn(process.hrtime, "bigint").mockImplementation(() => ++mono); try { const f = fixture(); f.events[4]!.createdAt = date(1); let dispatched = false; - const sample = vi.fn(async () => {}), post = vi.fn(async () => { dispatched = true; }); + const sample = vi.fn(async () => {}), post = stopPost(f, () => { dispatched = true; }); const retain = vi.fn(async (receipt) => { expect(receipt.terminal).toBeNull(); }); const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 3000, - load: async () => ({ ...f, events: dispatched ? f.events : f.events.filter((_, i) => i !== 4), retired: dispatched }), + load: async () => ({ ...f, run: dispatched ? f.run : liveRun(f), events: dispatched ? f.events : f.events.filter((_, i) => i !== 4), retired: dispatched }), afterDeniedEdit: async () => {}, retainPreStop: retain, afterSettlement: sample }); const rejected = expect(result).rejects.toThrow(/Timed out waiting for correlated provider settlement/); await vi.advanceTimersByTimeAsync(3200); await rejected; @@ -167,13 +177,13 @@ describe("Copilot denial provider settlement and separate audited run Stop", () it.each(["missing-edit", "foreign-edit", "missing-terminal", "foreign-terminal", "live-process"])("fails closed on %s without premature side effects or terminal samples", async kind => { vi.useFakeTimers(); let mono = 100n; vi.spyOn(process.hrtime, "bigint").mockImplementation(() => ++mono); try { - const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {}); + const f = fixture(), post = stopPost(f), sample = vi.fn(async () => {}); if (kind === "missing-edit") f.events.splice(3, 1); if (kind === "foreign-edit") f.events[3]!.payload.prpEvent.normalizedSessionId = "foreign"; if (kind === "missing-terminal") f.events.splice(4, 1); if (kind === "foreign-terminal") terminal(f).sourceInstanceId = "foreign"; const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 3000, - load: async () => ({ ...f, retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: sample }); + load: async () => ({ ...f, run: post.mock.calls.length ? f.run : liveRun(f), retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: sample }); const rejected = expect(result).rejects.toThrow(); await vi.advanceTimersByTimeAsync(3200); await rejected; expect(post).toHaveBeenCalledTimes(kind.endsWith("edit") || kind === "foreign-terminal" ? 0 : 1); @@ -181,31 +191,59 @@ describe("Copilot denial provider settlement and separate audited run Stop", () } finally { vi.restoreAllMocks(); vi.useRealTimers(); } }); it.each(["failed", "timed_out"])("reports %s promptly even when required rows are missing", async status => { - const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {}); + const f = fixture(), post = stopPost(f), sample = vi.fn(async () => {}); const load = vi.fn(async () => ({ ...f, events: [], run: { ...f.run, status }, retired: false })); await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 10000, load, afterDeniedEdit: sample, retainPreStop: sample, afterSettlement: sample })).rejects.toThrow(`Stopped waiting for persisted correlated failed native edit: Copilot provider run failed`); expect(load).toHaveBeenCalledOnce(); expect(post).not.toHaveBeenCalled(); expect(sample).not.toHaveBeenCalled(); }); it.each(["failed", "timed_out"])("reports post-Stop %s before attempting a missing terminal reader", async status => { - const f = fixture(), post = vi.fn(async () => {}); let loads = 0; - const load = vi.fn(async () => ({ ...f, events: ++loads < 3 ? f.events : [], run: { ...f.run, status: loads < 3 ? "running" : status }, retired: true })); + const f = fixture(), post = stopPost(f); let loads = 0; + const load = vi.fn(async () => ({ ...f, events: ++loads < 4 ? f.events : [], run: loads < 4 ? liveRun(f) : { ...f.run, status }, retired: true })); await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 10000, load, afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {} })).rejects.toThrow("Stopped waiting for correlated provider settlement and retired run: Copilot provider run failed"); - expect(load).toHaveBeenCalledTimes(3); expect(post).toHaveBeenCalledOnce(); + expect(load).toHaveBeenCalledTimes(4); expect(post).toHaveBeenCalledOnce(); }); it("does not dispatch Stop if the pre-dispatch receipt cannot be retained", async () => { - const f = fixture(), post = vi.fn(async () => {}); + const f = fixture(), post = stopPost(f); await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, - load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {}, + load: async () => ({ ...f, run: post.mock.calls.length ? f.run : liveRun(f), retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {}, retainPreStop: async () => { throw new Error("artifact write failed"); } })).rejects.toThrow("artifact write failed"); expect(post).not.toHaveBeenCalled(); }); it("retains completed-before-Stop settlement and propagates retirement sampling failure", async () => { - const f = fixture(), post = vi.fn(async () => {}); + const f = fixture(), post = stopPost(f); const input = { api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, - load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {} }; + load: async () => ({ ...f, run: post.mock.calls.length ? f.run : liveRun(f), retired: true }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {} }; expect((await settleCopilotDeniedRun(input)).branch).toBe("provider_completed_observed_before_stop"); + post.mockClear(); await expect(settleCopilotDeniedRun({ ...input, afterSettlement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live"); }); + it.each(["cancelled", "pending-intent", "startup-only"])("rejects visible earlier Stop %s before dispatch", async kind => { + const f = fixture(), post = stopPost(f); + const run = kind === "cancelled" ? f.run : { ...liveRun(f), resultJson: kind === "startup-only" + ? { startupCancellation: { requestedAt: date(1) } } : { nativeCancellation: { intentId: "earlier", dispatchState: "pending" } } }; + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, + load: async () => ({ ...f, run, retired: false }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {}, + })).rejects.toThrow("earlier Stop"); + expect(post).not.toHaveBeenCalled(); + }); + it("rejects Stop racing during artifact retention", async () => { + const f = fixture(), post = stopPost(f); let earlier = false; + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, + load: async () => ({ ...f, run: earlier ? f.run : liveRun(f), retired: false }), afterDeniedEdit: async () => {}, + retainPreStop: async () => { earlier = true; }, afterSettlement: async () => {}, + })).rejects.toThrow("earlier Stop"); + expect(post).not.toHaveBeenCalled(); + }); + it.each(["conflict", "foreign-response"])("cannot pass a Stop racing after the last API read: %s", async kind => { + const f = fixture(); const post = vi.fn(async (_path: string, _body: { cancellationRequestId: string }) => { if (kind === "conflict") throw new Error("409 cancellation intent conflict"); return f.run; }); + const sampled = vi.fn(async () => {}); + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, + load: async () => ({ ...f, run: liveRun(f), retired: false }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: sampled, + })).rejects.toThrow(kind === "conflict" ? "409" : "foreign cancellation intent"); + expect(post).toHaveBeenCalledOnce(); expect(sampled).not.toHaveBeenCalled(); + expect(post.mock.calls[0]?.[1]).toMatchObject({ cancellationRequestId: expect.stringMatching(/^[a-f0-9-]{36}$/) }); + }); + });