diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index b3a453458e8..a23524e0445 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3367,3 +3367,29 @@ checkpoints publish the metadata without a separate migration write. Disk-free rollback requires a server that reads this envelope and the supported settled warm journal, including its PRP v2 idle goal suffix. Earlier local-only readers and readers lacking that suffix handling remain outside that rollback contract. + +### Stop a failed native retry + +A failed native run can still own a scheduled same-run bootstrap retry. The +ordinary run detail offers Stop when the server's batched execution projection +sets `canStop`: the native run is failed, its matching coordinator is retryable, +and no result has been accepted. Unknown, legacy and accepted-result states +remain ineligible. Grok failed retries keep their existing external Stop contract +and do not gain this action; a pre-assignment failed retry is not supported by +that endpoint. Existing Grok running/queued Stop and assignment fences stay in +place. A false or absent hint does not remove the existing queued +or running Cancel action. This presentation hint grants no permission; the existing +company, actor and locked cancellation checks remain authoritative. A changed +retry can reject Stop, which the run detail displays without claiming cleanup. +Failed-native Stop keeps its request identity by company, run and current board +actor through navigation or reload. It uses the shared HTTP-compatible UUID +helper. A fresh run read can recover that same actor's canonical saved claim; +another actor's claim cannot be borrowed. Browser storage and in-memory fallback +retain unresolved request identities. This never retries Stop automatically. +The server locks and checks the current retry before it +accepts that identity. A stale request returns a conflict before cancellation +activity is recorded. The client also rejects an unchanged successful HTTP +response with no matching cancellation intent. A matching durable pending intent +can be accepted without a terminal acknowledgement. The selected run and its +history refresh after either outcome. Stop does not resume the provider or change +a manual task hold. diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index fd156eafc6f..fee5ade4c8a 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -28,6 +28,7 @@ export interface ExecutionBlocker { /** Presentation of existing execution records, not a second task status machine. */ export interface ExecutionProjection { + canStop?: boolean; phase: | "working" | "waiting_for_provider" diff --git a/server/src/__tests__/dot-runner.test.ts b/server/src/__tests__/dot-runner.test.ts index 53555d8f313..330c1a24c8a 100644 --- a/server/src/__tests__/dot-runner.test.ts +++ b/server/src/__tests__/dot-runner.test.ts @@ -589,6 +589,21 @@ describe("durable Dot Runner integration", () => { } finally { await f.events.stop(); } }, 30000); + it("does not authorize a failed Grok bootstrap retry without an external assignment", async () => { + const f = await fixture({ provider: "grok_bot" }); + try { + const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Bootstrap retry", assigneeAgentId: f.agent.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, + nativeIssueId: task.id, runtimeMode: "native", driverKind: "grok_bot_mcp", status: "failed", invocationSource: "assignment" }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, issueId: task.id, runId: run.id, phase: "retryable_failure" }); + const cancelRun = vi.fn().mockResolvedValue(null); + const broker = dotRunnerBroker(db, { heartbeat: { wakeup: vi.fn(), cancelRun } }); + await expect(broker.stopGrokWork(f.company.id, f.agent.id, f.userId, run.id)).rejects.toThrow("changed"); + expect(cancelRun).not.toHaveBeenCalled(); + expect(await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, run.id))).toHaveLength(0); + } finally { await f.events.stop(); } + }); + it("fences a failed Grok run's remaining external assignment without stopping a successor", async () => { const f = await fixture({ provider: "grok_bot" }); const old = await offeredWork(f); diff --git a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts index a610b5b25bd..cfbf7e79ed0 100644 --- a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts +++ b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts @@ -18,6 +18,7 @@ import { heartbeatRuns, issueAccessGrants, issues, + nativeRunFinalizations, toolApplications, toolConnections, workspaceOperations, @@ -33,6 +34,7 @@ import { agentRoutes } from "../routes/agents.js"; import { authorizationService } from "../services/authorization.js"; import { canActorReadWorkspaceOperation, getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { claimCancellationRequest } from "../services/native-runtime/native-cancellation-request.js"; const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL"); const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -56,6 +58,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, await db.delete(heartbeatRunEvents); await db.delete(issueAccessGrants); await db.delete(agentRuntimeState); + await db.delete(nativeRunFinalizations); await db.delete(heartbeatRuns); await db.delete(issues); await db.delete(agents); @@ -451,6 +454,54 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, expect(response.body).toEqual([]); }); + it("returns an actor-bound saved native Stop claim through the real run detail", async () => { + const fixture = await seedFixture(); + await db.update(heartbeatRuns).set({ status: "failed", runtimeMode: "native", nativeIssueId: fixture.issueId, resultJson: null }) + .where(eq(heartbeatRuns.id, fixture.privateRunId)); + await db.insert(nativeRunFinalizations).values({ runId: fixture.privateRunId, companyId: fixture.companyId, + issueId: fixture.issueId, phase: "retryable_failure" }); + const requestId = randomUUID(); + await claimCancellationRequest(db, fixture.privateRunId, fixture.companyId, requestId, fixture.ownerUserId); + const detail = await request(createBoardApp(fixture.companyId, fixture.ownerUserId)).get(`/api/heartbeat-runs/${fixture.privateRunId}`); + expect(detail.status).toBe(200); + expect(detail.body.execution.canStop).toBe(true); + expect(detail.body.resultJson.startupCancellation).toMatchObject({ cancellationRequestId: requestId, + requestedBy: { type: "board", userId: fixture.ownerUserId } }); + expect(detail.body.resultJson.nativeCancellation).toBeUndefined(); + await expect(claimCancellationRequest(db, fixture.privateRunId, fixture.companyId, requestId, fixture.otherUserId)) + .rejects.toMatchObject({ status: 409 }); + await expect(claimCancellationRequest(db, fixture.privateRunId, fixture.companyId, requestId, fixture.ownerUserId)) + .resolves.toMatchObject({ status: "failed" }); + }); + + it("rejects stale failed-native Stop correlation before cancellation activity is recorded", async () => { + const fixture = await seedFixture(); + await db.update(heartbeatRuns).set({ status: "failed", runtimeMode: "native", nativeIssueId: fixture.issueId, + driverKind: "codex_app_server", resultJson: null, + }).where(eq(heartbeatRuns.id, fixture.privateRunId)); + await db.insert(nativeRunFinalizations).values({ runId: fixture.privateRunId, companyId: fixture.companyId, + issueId: fixture.issueId, phase: "retryable_failure" }); + const app = createBoardApp(fixture.companyId, fixture.ownerUserId); + const detail = await request(app).get(`/api/heartbeat-runs/${fixture.privateRunId}`); + expect(detail.status).toBe(200); + expect(detail.body.execution.canStop).toBe(true); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "retry_authority_revoked" }) + .where(eq(nativeRunFinalizations.runId, fixture.privateRunId)); + const before = await db.select().from(activityLog).where(eq(activityLog.companyId, fixture.companyId)); + const unchanged = await request(app).post(`/api/heartbeat-runs/${fixture.privateRunId}/cancel`).send({}); + expect(unchanged.status).toBe(200); + expect(unchanged.body).toMatchObject({ status: "failed", resultJson: null }); + const recorded = await db.select().from(activityLog).where(eq(activityLog.companyId, fixture.companyId)); + expect(recorded).toHaveLength(before.length + 1); + expect(recorded.at(-1)?.action).toBe("heartbeat.cancelled"); + const denied = await request(app).post(`/api/heartbeat-runs/${fixture.privateRunId}/cancel`) + .send({ cancellationRequestId: randomUUID() }); + expect(denied.status).toBe(409); + expect(await db.select().from(activityLog).where(eq(activityLog.companyId, fixture.companyId))).toEqual(recorded); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.privateRunId)))[0]) + .toMatchObject({ status: "failed", resultJson: null }); + }); + it("conceals private runs from unauthorized cancellation and watchdog mutations", async () => { const fixture = await seedFixture(); diff --git a/server/src/services/execution-projection-recovery.test.ts b/server/src/services/execution-projection-recovery.test.ts index a05c868d864..5f1628d6a96 100644 --- a/server/src/services/execution-projection-recovery.test.ts +++ b/server/src/services/execution-projection-recovery.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, completionContracts } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { executionProjectionsForRuns } from "./execution-projection.js"; @@ -36,4 +37,42 @@ describe.skipIf(!support.supported)("persisted stopped-workspace projection", () expect(projections.get(runId)).toMatchObject({ phase: "recovery_needed", label: "Stopped", nextAction: "Saved work recovered. Send a message to continue.", permittedActions: ["inspect_run"] }); }); + it("advertises Stop for a failed native retry from the real batched coordinator read", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Pending native retry", issuePrefix: "PNR" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Pending retry", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", finishedAt: new Date(), contextSnapshot: { issueId } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "retryable_failure", resultId: null }); + const projections = await executionProjectionsForRuns(db, companyId, [runId]); + expect(projections.get(runId)).toMatchObject({ canStop: true }); + await db.update(heartbeatRuns).set({ driverKind: "grok_bot_mcp" }).where(eq(heartbeatRuns.id, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + await db.update(heartbeatRuns).set({ driverKind: "codex_app_server" }).where(eq(heartbeatRuns.id, runId)); + for (const status of ["succeeded", "cancelled", "timed_out"] as const) { + await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + } + await db.update(heartbeatRuns).set({ status: "failed", runtimeMode: "legacy" }).where(eq(heartbeatRuns.id, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + for (const phase of ["observed", "terminal_failure", "committed"] as const) { + await db.update(nativeRunFinalizations).set({ phase }).where(eq(nativeRunFinalizations.runId, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + } + await db.update(nativeRunFinalizations).set({ phase: "retryable_failure" }).where(eq(nativeRunFinalizations.runId, runId)); + const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "low", completionAuthority: "server", + incompleteCriteriaPolicy: "deny", contractJson: {}, canonicalSha256: randomUUID(), createdByActorType: "agent", createdByActorId: agentId }).returning(); + await db.update(heartbeatRuns).set({ completionContractId: contract.id }).where(eq(heartbeatRuns.id, runId)); + const [accepted] = await db.insert(nativeRunResults).values({ companyId, issueId, runId, completionContractId: contract.id, + schemaStatus: "accepted", serverFingerprint: randomUUID(), canonicalSha256: randomUUID(), resultJson: { summary: "Accepted result" } }).returning(); + await db.update(nativeRunFinalizations).set({ resultId: accepted.id }).where(eq(nativeRunFinalizations.runId, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + expect((await executionProjectionsForRuns(db, randomUUID(), [runId])).has(runId)).toBe(false); + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect((await executionProjectionsForRuns(db, companyId, [runId])).get(runId)?.canStop).toBe(false); + }); + }); diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 84845becfe7..1e7b5c65340 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -223,6 +223,9 @@ export function projectExecution( const detail = coordinator?.failureDetail ?? {}; const successorRunId = text(detail.successorRunId); const projection: ExecutionProjection = { + canStop: run.runtimeMode === "native" && run.status === "failed" && run.driverKind !== "grok_bot_mcp" && !!run.nativeIssueId + && coordinator?.runId === run.id && coordinator.issueId === run.nativeIssueId && coordinator.resultId === null + && coordinator.phase === "retryable_failure", phase: "queued", label: "Queued", cause: run.errorCode ?? coordinator?.failureCode ?? null, diff --git a/ui/src/api/heartbeats.ts b/ui/src/api/heartbeats.ts index 929072f43d6..9fd90b53702 100644 --- a/ui/src/api/heartbeats.ts +++ b/ui/src/api/heartbeats.ts @@ -216,8 +216,20 @@ export const heartbeatsApi = { }>( `/workspace-operations/${operationId}/log?offset=${encodeURIComponent(String(offset))}&limitBytes=${encodeURIComponent(String(limitBytes))}`, ), - cancel: (runId: string) => - api.post(`/heartbeat-runs/${runId}/cancel`, {}), + cancel: async (runId: string, options?: { cancellationRequestId: string }) => { + const run = await api.post(`/heartbeat-runs/${runId}/cancel`, options ?? {}); + if (options && run?.status !== "cancelled") { + const value = run?.resultJson?.nativeCancellation; + const intent = value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; + const pending = run?.id === runId && intent?.schema === "paperclip.native-cancellation.v1" + && intent.runId === runId && intent.companyId === run.companyId + && intent.intentId === `native-cancellation:${options.cancellationRequestId}` && intent.scope === "run" + && intent.dispatchState === "pending" && intent.dispatched === false + && typeof intent.intentAuditId === "string" && intent.intentAuditId.length > 0; + if (!pending) throw new Error("Stop was not accepted because the run changed."); + } + return run; + }, resolveRuntimeRequest: (input: { runId: string; requestId: string; diff --git a/ui/src/lib/native-run-stop.ts b/ui/src/lib/native-run-stop.ts new file mode 100644 index 00000000000..0b4e084b392 --- /dev/null +++ b/ui/src/lib/native-run-stop.ts @@ -0,0 +1,41 @@ +import { accessApi } from "../api/access"; +import { detachInflightGet } from "../api/client"; +import { heartbeatsApi } from "../api/heartbeats"; +import { clearPendingSend, pendingSendKeyFor, readPendingSend, writePendingSend, type PendingSend } from "./pending-send"; +import { createUuid } from "./uuid"; + +type StopScope = { runId: string; companyId: string; actorUserId: string }; +const memoryRequests = new Map>(); +const uuid = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i; +const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +export async function stopFailedNativeRun(runId: string, companyId: string) { + detachInflightGet(`/heartbeat-runs/${runId}`); + detachInflightGet("/cli-auth/me"); + const [run, access] = await Promise.all([heartbeatsApi.get(runId), accessApi.getCurrentBoardAccess()]); + if (run.id !== runId || run.companyId !== companyId) throw new Error("Stop was not accepted because the run changed."); + const actorUserId = access.userId; + if (!actorUserId) throw new Error("Stop was not accepted because your session changed."); + const key = pendingSendKeyFor(`paperclip:native-run-stop:${companyId}:${runId}`, actorUserId); + const matchesScope = (value: unknown): value is StopScope => { + const scope = record(value); + return scope.runId === runId && scope.companyId === companyId && scope.actorUserId === actorUserId; + }; + const startup = record(run.resultJson?.startupCancellation); + const canonicalId = startup.cancellationRequestId; + if (canonicalId !== undefined) { + const actor = record(startup.requestedBy); + if (actor.type !== "board" || actor.userId !== actorUserId) throw new Error("This Stop request belongs to another operator."); + if (typeof canonicalId !== "string" || !uuid.test(canonicalId)) throw new Error("Stop was not accepted because the run changed."); + } + const saved = readPendingSend(key, matchesScope, "session") ?? memoryRequests.get(key); + const id = typeof canonicalId === "string" ? canonicalId + : saved && matchesScope(saved.payload) && uuid.test(saved.idempotencyKey) ? saved.idempotencyKey : createUuid(); + const pending = { idempotencyKey: id, payload: { runId, companyId, actorUserId }, createdAt: saved?.createdAt ?? Date.now() }; + memoryRequests.set(key, pending); + writePendingSend(key, pending, "session"); + const result = await heartbeatsApi.cancel(runId, { cancellationRequestId: id }); + clearPendingSend(key, id, "session"); + if (memoryRequests.get(key)?.idempotencyKey === id) memoryRequests.delete(key); + return result; +} diff --git a/ui/src/lib/run-cancellation-label.ts b/ui/src/lib/run-cancellation-label.ts new file mode 100644 index 00000000000..86eace857c8 --- /dev/null +++ b/ui/src/lib/run-cancellation-label.ts @@ -0,0 +1,5 @@ +export function runCancellationLabel(input: { grokWork?: boolean; isFailed: boolean; isPending: boolean }): string { + if (input.grokWork) return input.isPending ? "Stopping Paperclip work…" : "Stop Paperclip work"; + if (input.isFailed) return input.isPending ? "Stopping…" : "Stop"; + return input.isPending ? "Cancelling…" : "Cancel"; +} diff --git a/ui/src/pages/AgentDetail.native-retry-stop.test.tsx b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx new file mode 100644 index 00000000000..b04d2eb9b25 --- /dev/null +++ b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx @@ -0,0 +1,170 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter } from "react-router-dom"; +import type { HeartbeatRun } from "@paperclipai/shared"; +import { afterEach, expect, it, vi } from "vitest"; +import { RunsTab } from "./AgentDetail"; +import { RunsTab as ProductionRunsTab } from "./AgentDetail.production"; +import { ToastProvider } from "../context/ToastContext"; +import { PanelProvider } from "../context/PanelContext"; +import { pendingSendKeyFor, readPendingSend, writePendingSend } from "../lib/pending-send"; + +const { get, cancel, grokStop, actor } = vi.hoisted(() => ({ get: vi.fn(), cancel: vi.fn(), grokStop: vi.fn(), actor: { id: "operator-1", source: "authenticated" } })); +vi.mock("../api/heartbeats", async importOriginal => ({ heartbeatsApi: { ...(await importOriginal()).heartbeatsApi, + get, events: async () => [], workspaceOperations: async () => [] } })); +vi.mock("../api/grokBot", () => ({ grokBotApi: { stop: grokStop } })); +vi.mock("../api/client", () => ({ api: { get: async () => [], post: cancel }, detachInflightGet: vi.fn() })); +vi.mock("../api/access", () => ({ accessApi: { getCurrentBoardAccess: async () => ({ userId: actor.id, source: actor.source, companyIds: ["company-1"], memberships: [] }) } })); +vi.mock("../context/SidebarContext", () => ({ useSidebar: () => ({ isMobile: false }) })); +vi.mock("../context/CompanyContext", () => ({ useCompany: () => ({ selectedCompanyId: "company-1", selectedCompany: { id: "company-1", issuePrefix: "TEST" }, companies: [] }) })); +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; +afterEach(() => { get.mockReset(); cancel.mockReset(); grokStop.mockReset(); actor.id = "operator-1"; actor.source = "authenticated"; sessionStorage.clear(); vi.unstubAllGlobals(); }); + +const states = [ + { status: "failed", eligibility: true, label: "Stop" }, + { status: "failed", eligibility: false, label: null }, + { status: "failed", eligibility: undefined, label: null }, + { status: "succeeded", eligibility: false, label: null }, + { status: "cancelled", eligibility: false, label: null }, + { status: "succeeded", eligibility: true, label: null }, + { status: "cancelled", eligibility: true, label: null }, + { status: "running", eligibility: undefined, label: "Cancel" }, + { status: "queued", eligibility: undefined, label: "Cancel" }, + { status: "failed", eligibility: true, label: "Stop", denial: true }, + { status: "failed", eligibility: true, label: "Stop", unchanged: true }, + { status: "failed", eligibility: true, label: "Stop", pending: true }, + { status: "failed", eligibility: true, label: "Stop", pending: true, wrongIntent: "run" }, + { status: "failed", eligibility: true, label: "Stop", pending: true, wrongIntent: "request" }, + { status: "failed", eligibility: true, label: "Stop", uncertain: true }, + { status: "failed", eligibility: true, label: "Stop", uncertain: true, remount: true }, + { status: "failed", eligibility: true, label: "Stop", uncertain: true, remount: true, canonical: true }, + { status: "failed", eligibility: true, label: "Stop", uncertain: true, remount: true, canonical: true, actorChanged: true }, + { status: "failed", eligibility: true, label: "Stop", uncertain: true, remount: true, canonical: true, wrongRun: true }, + { status: "failed", eligibility: true, label: "Stop", http: true }, + { status: "failed", eligibility: true, label: "Stop", existingClaim: true }, + { status: "failed", eligibility: true, label: "Stop", existingClaim: true, local: true }, + { status: "failed", eligibility: true, label: "Stop", priorSession: true }, + { status: "failed", eligibility: true, label: null, driverKind: "grok_bot_mcp" }, + { status: "running", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, + { status: "queued", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, +]; +it.each([RunsTab, ProductionRunsTab].flatMap(Tab => states.map(state => ({ Tab, ...state }))))("renders and submits only eligible run cancellation (%#)", async ({ Tab, status, eligibility, label, ...state }) => { + if (state.local) { actor.id = "local-board"; actor.source = "local_implicit"; } + const run = { id: state.priorSession ? `persisted-run-${Tab === RunsTab ? "main" : "production"}` : "failed-native-run", companyId: "company-1", agentId: "agent-1", issueId: "issue-1", status, driverKind: state.driverKind, invocationSource: "manual", + createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", usageJson: null, resultJson: null, + contextSnapshot: null, exitCode: 1, error: "Runner ingress unavailable", logRef: null, startedAt: null, finishedAt: null, + execution: { canStop: eligibility, phase: "reconnecting", label: "Reconnecting", recoveryOwner: "agent", retryAt: "2026-01-01T00:00:01Z" }, + } as unknown as HeartbeatRun; + const existingRequestId = "11111111-1111-4111-8111-111111111111"; + const storageKey = pendingSendKeyFor(`paperclip:native-run-stop:${run.companyId}:${run.id}`, actor.id); + if (state.priorSession) expect(writePendingSend(storageKey, { idempotencyKey: existingRequestId, + payload: { runId: run.id, companyId: run.companyId, actorUserId: actor.id }, createdAt: Date.now() - 1000, + }, "session")).toBe(true); + if (state.existingClaim) run.resultJson = { startupCancellation: { cancellationRequestId: existingRequestId, + requestedBy: { type: "board", userId: actor.id } } }; + get.mockResolvedValue(run); + if (state.http) vi.stubGlobal("crypto", { getRandomValues: globalThis.crypto.getRandomValues.bind(globalThis.crypto) }); + cancel.mockImplementation(async (_path, body) => { + if (state.denial) throw new Error("Native retry changed; Stop was not accepted"); + if (state.uncertain && cancel.mock.calls.length === 1) { + if (state.canonical) get.mockResolvedValue({ ...run, resultJson: { startupCancellation: { + cancellationRequestId: body.cancellationRequestId, requestedBy: { type: "board", userId: "operator-1" }, + } } }); + throw new Error("Stop outcome is unknown"); + } + if (state.unchanged) { + const unchanged = { ...run, execution: { ...run.execution, canStop: false } }; + get.mockResolvedValue(unchanged); + return unchanged; + } + if (state.pending || state.uncertain) { + const pending = { ...run, execution: { ...run.execution, canStop: false }, resultJson: { nativeCancellation: { + schema: "paperclip.native-cancellation.v1", runId: state.wrongIntent === "run" ? "another-run" : run.id, companyId: run.companyId, issueId: run.issueId, + intentId: `native-cancellation:${state.wrongIntent === "request" ? "another-request" : body.cancellationRequestId}`, scope: "run", dispatchState: "pending", dispatched: false, intentAuditId: "recorded-intent", + } } }; + get.mockResolvedValue(pending); + return pending; + } + const stopped = { ...run, status: "cancelled", execution: { ...run.execution, canStop: false } }; + get.mockResolvedValue(stopped); + return stopped; + }); + grokStop.mockImplementation(async () => { + get.mockResolvedValue({ ...run, status: "cancelled", execution: { ...run.execution, canStop: false } }); + return { stopped: true, remoteTerminationConfirmed: false }; + }); + const usesGrokStop = Tab === RunsTab && state.driverKind === "grok_bot_mcp"; + const expectedLabel = label && usesGrokStop ? "Stop Paperclip work" : label; + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + const element = document.createElement("div"); document.body.append(element); + let root = createRoot(element); + const render = () => root.render( + + ); + try { + await act(async () => render()); + await vi.waitFor(async () => { await act(async () => {}); expect(element.textContent).toContain(status); }); + const control = () => Array.from(element.querySelectorAll("button")).find(candidate => ["Stop", "Cancel", "Stop Paperclip work"].includes(candidate.textContent ?? "")); + if (label) { + expect(control()?.textContent).toBe(expectedLabel); + await act(async () => control()!.click()); + await vi.waitFor(() => usesGrokStop + ? expect(grokStop).toHaveBeenCalledExactlyOnceWith("company-1", "agent-1", run.id) + : expect(cancel).toHaveBeenCalledExactlyOnceWith(`/heartbeat-runs/${run.id}/cancel`, + status === "failed" ? { cancellationRequestId: expect.stringMatching(/^[a-f0-9-]{36}$/) } : {})); + expect(usesGrokStop ? cancel : grokStop).not.toHaveBeenCalled(); + if (state.existingClaim || state.priorSession) expect(cancel.mock.calls[0][1]).toEqual({ cancellationRequestId: existingRequestId }); + if (state.uncertain) { + await vi.waitFor(async () => { + await act(async () => {}); + expect(element.querySelector('[role="alert"]')?.textContent).toContain("Stop outcome is unknown"); + expect(control()?.textContent).toBe(expectedLabel); + }); + const firstRequest = cancel.mock.calls[0][1]; + expect(readPendingSend(storageKey, (value): value is Record => typeof value === "object" && value !== null, "session")) + .toMatchObject({ idempotencyKey: firstRequest.cancellationRequestId, + payload: { runId: run.id, companyId: run.companyId, actorUserId: actor.id } }); + if (state.remount) { + await act(async () => root.unmount()); + client.clear(); + if (state.canonical) sessionStorage.clear(); + if (state.actorChanged) actor.id = "operator-2"; + root = createRoot(element); + await act(async () => render()); + await vi.waitFor(async () => { await act(async () => {}); expect(control()?.textContent).toBe(expectedLabel); }); + if (state.wrongRun) get.mockResolvedValue({ ...run, id: "another-run", resultJson: { startupCancellation: { + cancellationRequestId: firstRequest.cancellationRequestId, requestedBy: { type: "board", userId: "operator-1" }, + } } }); + } + await act(async () => control()!.click()); + if (state.actorChanged || state.wrongRun) { + await vi.waitFor(async () => { + await act(async () => {}); + expect(element.querySelector('[role="alert"]')?.textContent).toContain(state.actorChanged ? "another operator" : "unavailable for this agent"); + }); + expect(cancel).toHaveBeenCalledTimes(1); + return; + } + await vi.waitFor(() => expect(cancel).toHaveBeenCalledTimes(2)); + expect(cancel.mock.calls[1][1]).toEqual(firstRequest); + } + await vi.waitFor(async () => { + await act(async () => {}); + if (state.denial || state.unchanged || state.wrongIntent) { + expect(element.querySelector('[role="alert"]')?.textContent).toContain("Stop was not accepted"); + if (state.denial) expect(control()?.textContent).toBe(expectedLabel); + } else { + expect(control()).toBeUndefined(); + expect(element.textContent).toContain(state.pending || state.uncertain ? "failed" : "cancelled"); + expect(element.querySelector('[role="alert"]')).toBeNull(); + } + }); + } else { + expect(control()).toBeUndefined(); + expect(cancel).not.toHaveBeenCalled(); + expect(grokStop).not.toHaveBeenCalled(); + } + } finally { await act(async () => root.unmount()); client.clear(); element.remove(); } +}); diff --git a/ui/src/pages/AgentDetail.production.tsx b/ui/src/pages/AgentDetail.production.tsx index c692d065f2f..c9eaa12419a 100644 --- a/ui/src/pages/AgentDetail.production.tsx +++ b/ui/src/pages/AgentDetail.production.tsx @@ -14,6 +14,8 @@ import { builtInAgentsApi, type BuiltInManagedResourceKind } from "../api/builtI import { companySkillsApi } from "../api/companySkills"; import { budgetsApi } from "../api/budgets"; import { heartbeatsApi } from "../api/heartbeats"; +import { runCancellationLabel } from "../lib/run-cancellation-label"; +import { stopFailedNativeRun } from "../lib/native-run-stop"; import { instanceSettingsApi } from "../api/instanceSettings"; import { ApiError } from "../api/client"; import { ChartCard, RunActivityChart, PriorityChart, IssueStatusChart, SuccessRateChart } from "../components/ActivityCharts"; @@ -2804,8 +2806,12 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } }, [run.id]); const cancelRun = useMutation({ - mutationFn: () => heartbeatsApi.cancel(run.id), - onSuccess: () => { + mutationFn: () => { + if (run.status !== "failed") return heartbeatsApi.cancel(run.id); + return stopFailedNativeRun(initialRun.id, initialRun.companyId); + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.runDetail(run.id) }); queryClient.invalidateQueries({ queryKey: queryKeys.heartbeats(run.companyId, run.agentId) }); }, }); @@ -2930,7 +2936,7 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig }
- {(run.status === "running" || run.status === "queued") && ( + {(run.status === "running" || run.status === "queued" || (run.status === "failed" && run.driverKind !== "grok_bot_mcp" && run.execution?.canStop === true)) && ( )} {canResumeLostRun && ( @@ -2998,6 +3004,11 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig }
)} + {cancelRun.error && ( +
+ {cancelRun.error instanceof Error ? cancelRun.error.message : "Failed to stop run"} +
+ )} {resumeRun.isError && ( // query-error-ok: mutation result, not a query
{resumeRun.error instanceof Error ? resumeRun.error.message : "Failed to resume run"} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index 575d92b67e5..3245c5a384b 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -22,6 +22,8 @@ import { import { builtInAgentsApi, type BuiltInManagedResourceKind } from "../api/builtInAgents"; import { companySkillsApi } from "../api/companySkills"; import { heartbeatsApi } from "../api/heartbeats"; +import { runCancellationLabel } from "../lib/run-cancellation-label"; +import { stopFailedNativeRun } from "../lib/native-run-stop"; import { instanceSettingsApi } from "../api/instanceSettings"; import { ApiError } from "../api/client"; import { activityApi } from "../api/activity"; @@ -2960,8 +2962,13 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } const grokWork = run.driverKind === "grok_bot_mcp"; const cancelRun = useMutation({ - mutationFn: () => grokWork ? grokBotApi.stop(run.companyId, run.agentId, run.id) : heartbeatsApi.cancel(run.id), - onSuccess: () => { + mutationFn: () => { + if (grokWork) return grokBotApi.stop(run.companyId, run.agentId, run.id); + if (run.status !== "failed") return heartbeatsApi.cancel(run.id); + return stopFailedNativeRun(initialRun.id, initialRun.companyId); + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.runDetail(run.id) }); queryClient.invalidateQueries({ queryKey: queryKeys.heartbeats(run.companyId, run.agentId) }); }, }); @@ -3125,7 +3132,7 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } requested={runRequestedProviderTrace(run.contextSnapshot)} showOff /> - {(run.status === "running" || run.status === "queued") && ( + {(run.status === "running" || run.status === "queued" || (run.status === "failed" && run.driverKind !== "grok_bot_mcp" && run.execution?.canStop === true)) && ( )} {canResumeLostRun && ( @@ -3241,6 +3248,11 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} + {cancelRun.error && ( +
+ {cancelRun.error instanceof Error ? cancelRun.error.message : "Failed to stop run"} +
+ )} {resumeRun.isError && ( // query-error-ok: mutation result, not a query
{resumeRun.error instanceof Error ? resumeRun.error.message : "Failed to resume run"}