diff --git a/docs/api/issues.md b/docs/api/issues.md index b4f0a6645ca..d7425507ec1 100644 --- a/docs/api/issues.md +++ b/docs/api/issues.md @@ -172,6 +172,24 @@ POST /api/issues/{issueId}/release Releases your ownership of the task. +## Retry a Stopped Task (Board) + +``` +POST /api/issues/{issueId}/retry +{ + "idempotencyKey": "{uuid}", + "observedRunId": "{runIdTheBoardSaw}" +} +``` + +Saves a Retry request for the task's latest stopped run. The same `idempotencyKey` returns the same request. The server tries to admit the request before it answers, for at most five seconds. + +- `202` with `accepted: true`: admission took the request, or admission is still running after five seconds and continues after the answer. +- `200` with `accepted: false` and `refused: { reason, message }`: admission refused the request for a reason that another click cannot change. `message` is the text that the task view shows. The request stays saved. If its reason clears later, it starts by itself. +- `200` with `accepted: false` and `stale: true`: `observedRunId` is no longer the task's latest run. Read the task again. + +Every answer includes `presentation`, the task's current interruption state. + ## Comments ### List Comments diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index b8e2cb9fbd0..73f044cb8fa 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -66884,6 +66884,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { app.use((req, _res, next) => { req.actor = boardActor(companyId); next(); }); app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service })); app.use(errorHandler); + // The route admits a plain Retry before it answers. This test checks + // the saved intent, not dispatch: a paused agent keeps it unstarted. + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); const continued = await request(app).post(`/api/issues/${issueId}/retry`).send({ idempotencyKey: input.idempotencyKey, observedRunId: input.observedRunId, }).expect(202); diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts index b7f16117cab..c5e7342b877 100644 --- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts +++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts @@ -192,6 +192,7 @@ function registerRouteMocks() { vi.doMock("../services/task-interruption-presentation.js", () => ({ taskInterruptionPresentation: vi.fn(async () => null), taskInterruptionPresentations: vi.fn(async () => new Map()), + taskRetryRefusal: vi.fn(async () => null), })); vi.doMock("../services/native-runtime/native-workspace-owner-recovery.js", () => ({ resumeNativeWorkspaceAfterOwnerStop: mockResumeWorkspaceFinalization })); vi.doMock("../services/native-runtime/native-workspace-export-retry.js", () => ({ retryNativeWorkspaceExport: mockRetryWorkspaceExport })); diff --git a/server/src/__tests__/issues-goal-context-routes.test.ts b/server/src/__tests__/issues-goal-context-routes.test.ts index b3cb0e6c41c..133b110797b 100644 --- a/server/src/__tests__/issues-goal-context-routes.test.ts +++ b/server/src/__tests__/issues-goal-context-routes.test.ts @@ -109,6 +109,7 @@ const mockDb = vi.hoisted(() => ({ vi.mock("../services/task-interruption-presentation.js", () => ({ taskInterruptionPresentation: vi.fn(async () => null), taskInterruptionPresentations: vi.fn(async () => new Map()), + taskRetryRefusal: vi.fn(async () => null), })); vi.mock("../services/index.js", () => ({ diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index f49b584b194..d30fe1881fa 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -12,7 +12,7 @@ import { readInboxMine } from "../services/inbox-mine-read.js"; import { subscribeCompanyLiveEvents } from "../services/live-events.js"; import { INBOX_MINE_ISSUE_STATUS_FILTER, latestFailedRunsByAgent } from "@paperclipai/shared"; import { getHeartbeatRunReadDecisions, redactHeartbeatRunListRow } from "../services/heartbeat-run-privacy.js"; -import { taskInterruptionPresentation, taskInterruptionPresentations } from "../services/task-interruption-presentation.js"; +import { taskInterruptionPresentation, taskInterruptionPresentations, taskRetryRefusal } from "../services/task-interruption-presentation.js"; import { parseObject } from "../adapters/utils.js"; import { prepareRestrictedWorkspaceRetry } from "../services/restricted-workspace-retry.js"; import { ISSUE_SUMMARIES_MAX_IDS, issueChatBootstrapQuerySchema, issueSummariesQuerySchema, type IssueSummary } from "@paperclipai/shared"; @@ -3626,6 +3626,9 @@ async function recordAgentChildTaskRead(db: Db, req: Request, issue: { id: strin .catch(error => logger.warn({ err: error, runId: req.actor.runId }, "could not record the child task a run read")); } +/** How long a Retry answer waits for admission before it reports the saved request as accepted. */ +const TASK_RETRY_ANSWER_ADMISSION_WAIT_MS = 5_000; + export function issueRoutes( db: Db, storage: StorageService, @@ -10433,13 +10436,39 @@ export function issueRoutes( runId: scheduledRetry.scheduledRetry?.runId ?? scheduledRetryRunId ?? null, details: { outcome: scheduledRetry.outcome, message: scheduledRetry.message, source: "task_retry" } }); } - res.status(receipt.accepted ? 202 : 200).json({ ...receipt, - ...(scheduledRetry ? { scheduledRetry: { outcome: scheduledRetry.outcome, message: scheduledRetry.message } } : {}), - presentation: await taskInterruptionPresentation(db, issue) }); - // The receipt commits before dispatch. Maintenance redelivers it after a - // crash or a lost response; this nudge only reduces perceived latency. const chatActionId = "chatActionId" in receipt ? receipt.chatActionId : undefined; const finalizeRunId = "finalizeRunId" in receipt ? receipt.finalizeRunId : undefined; + // Admit a plain Retry before answering, so the answer never reports as + // accepted a request that admission refused for a reason another click + // cannot change. The receipt is already committed. Admission that waits + // (for example on the agent's start lock) keeps running after the answer, + // which then stays "accepted"; maintenance redelivers it after a crash. + let refused: { reason: string; message: string } | null = null; + let admissionSettled: string | null = null; + if (receipt.accepted && receipt.requestId && !finalizeRunId && !chatActionId) { + const requestId = receipt.requestId; + const admitted = heartbeat.resumeTaskRetry(issue.companyId, requestId).then(() => true, err => { + logger.warn({ err, issueId: issue.id }, "Task Retry remains queued"); + return false; + }); + let timer: NodeJS.Timeout | undefined; + const answeredInTime = await Promise.race([admitted, new Promise(resolve => { + timer = setTimeout(() => resolve(false), TASK_RETRY_ANSWER_ADMISSION_WAIT_MS); + timer.unref?.(); + })]); + if (timer) clearTimeout(timer); + if (answeredInTime) admissionSettled = requestId; + } + const presentation = await taskInterruptionPresentation(db, issue); + if (admissionSettled) refused = await Promise.resolve() + .then(() => taskRetryRefusal(db, issue.companyId, admissionSettled!, presentation)).catch(err => { + logger.warn({ err, issueId: issue.id }, "Task Retry admission state could not be read"); + return null; + }); + res.status(receipt.accepted && !refused ? 202 : 200).json({ ...receipt, + ...(refused ? { accepted: false, refused } : {}), + ...(scheduledRetry ? { scheduledRetry: { outcome: scheduledRetry.outcome, message: scheduledRetry.message } } : {}), + presentation }); if (finalizeRunId) { // The saved result is applied again; no provider run starts. The // result-commit worker repeats this if the process stops first. @@ -10448,8 +10477,6 @@ export function issueRoutes( logger.warn({ err, issueId: issue.id, runId: finalizeRunId }, "Saved result finalization remains queued")); } else if (chatActionId) void opts.chatRunRetries?.processFailedChatRunRetry(chatActionId).catch(err => logger.warn({ err, issueId: issue.id }, "Connected task Retry remains queued")); - else if (receipt.accepted && receipt.requestId) void heartbeat.resumeTaskRetry(issue.companyId, receipt.requestId).catch(err => - logger.warn({ err, issueId: issue.id }, "Task Retry remains queued")); }); router.post("/issues/:id/recovery-actions/recover-workspace", validate(recoverWorkspaceSchema), async (req, res) => { diff --git a/server/src/services/legacy-preparation-retry-hold.test.ts b/server/src/services/legacy-preparation-retry-hold.test.ts index 3f3fe9dab5d..3acc315b509 100644 --- a/server/src/services/legacy-preparation-retry-hold.test.ts +++ b/server/src/services/legacy-preparation-retry-hold.test.ts @@ -1,8 +1,12 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, agentWakeupRequests, companies, createDb, environmentLeases, executionWorkspaces, +import express from "express"; +import request from "supertest"; +import { agents, agentWakeupRequests, companies, companyMemberships, createDb, environmentLeases, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { errorHandler } from "../middleware/index.js"; +import { issueRoutes } from "../routes/issues.js"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { requestTaskRetry } from "./task-retry.js"; import { heartbeatService } from "./heartbeat.js"; @@ -151,4 +155,33 @@ describe("Retry of a legacy preparation that automatic recovery stopped", () => expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")))).toHaveLength(0); }); + + // The Retry answer must never say "accepted" for a request that admission + // refuses for a reason another click cannot change. Here the stopped + // preparation recorded a provider event, so its outcome is really unknown. + it("answers a refused Retry with its refusal instead of accepted, and keeps the request saved", async () => { + const f = await seed(); + await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, agentId: f.agentId, + runId: f.source.id, seq: 5, eventType: "tool.execution.started", stream: "system" }); + await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: "board-user", + status: "active", membershipRole: "admin" }); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", userId: "board-user", companyIds: [f.companyId], + memberships: [{ companyId: f.companyId, membershipRole: "admin", status: "active" }], + isInstanceAdmin: false, source: "session" } as Express.Request["actor"]; + next(); + }); + app.use("/api", issueRoutes(db, {} as never)); + app.use(errorHandler); + const res = await request(app).post(`/api/issues/${f.issueId}/retry`) + .send({ idempotencyKey: randomUUID(), observedRunId: f.source.id }); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ accepted: false, requestId: expect.any(String), + refused: { reason: "previous_execution_unverified", message: expect.stringContaining("can't confirm") } }); + // The saved request still starts by itself if the gate clears later. + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, res.body.requestId)); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); + }, 60_000); }); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 47ed6119208..9374d564e6e 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -174,6 +174,11 @@ const retryWaitMessages: Record = { retry_source_changed: "Retry cannot continue because its saved message changed. Send a new message to continue from saved work.", }; +/** The stable text for a Retry admission refusal, as the task view shows it. */ +export function taskRetryWaitMessage(reason: string): string { + return retryWaitMessages[reason] ?? "Retry is waiting for execution checks. Your message is saved."; +} + const UNIDENTIFIED_RUNNER_STOP_MESSAGE = "Paperclip can't confirm that the previous run stopped. The server restarted after it started the agent but before it recorded the agent's process. " + "If no agent is still working on this task, a board operator can confirm that it stopped, and your Retry starts right away. Your message is saved."; @@ -900,3 +905,23 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i export async function taskInterruptionPresentation(db: Db, issue: ProjectionIssue) { return (await taskInterruptionPresentations(db, issue.companyId, [issue])).get(issue.id) ?? null; } + +/** + * The refusal that admission recorded on this Retry request, when another + * click cannot change it. A Retry answer must name it instead of reporting + * the request as accepted: such a request starts no run by itself. The task + * view's projection says whether a quarantined session is still cleaning up. + */ +export async function taskRetryRefusal(db: Pick, companyId: string, requestId: string, + presentation?: Pick | null): + Promise<{ reason: string; message: string } | null> { + const [wake] = await db.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId, + reason: sql`${agentWakeupRequests.payload}#>>'{executionWait,reason}'` }) + .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, requestId))); + if (!wake || wake.status !== "deferred_issue_execution" || wake.runId || !wake.reason) return null; + const shown = presentation?.retryWait?.reason === wake.reason ? presentation : null; + // A quarantined session that is still cleaning up is a refusal only while + // the task view hides Retry for it; every other reason has a fixed verdict. + if (heldRetryOffersRetry(wake.reason, wake.reason === "cleanup_quarantined" && shown?.canRetry === false)) return null; + return { reason: wake.reason, message: shown?.retryWait?.message ?? taskRetryWaitMessage(wake.reason) }; +} diff --git a/server/src/services/task-retry-refusal.test.ts b/server/src/services/task-retry-refusal.test.ts new file mode 100644 index 00000000000..9fac1206173 --- /dev/null +++ b/server/src/services/task-retry-refusal.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { taskRetryRefusal } from "./task-interruption-presentation.js"; + +function dbWithWake(wake: { status: string; runId: string | null; reason: string | null } | null) { + return { select: () => ({ from: () => ({ where: async () => (wake ? [wake] : []) }) }) } as unknown as Pick; +} + +const held = (reason: string) => dbWithWake({ status: "deferred_issue_execution", runId: null, reason }); + +describe("Retry refusal in the Retry answer", () => { + it("names a refusal that another click cannot change", async () => { + expect(await taskRetryRefusal(held("previous_execution_unverified"), "company", "request")) + .toMatchObject({ reason: "previous_execution_unverified", message: expect.stringContaining("can't confirm") }); + }); + + it("refuses a quarantined session's Retry while the task view says its cleanup is still settling", async () => { + const presentation = { canRetry: false, retryWait: { reason: "cleanup_quarantined", message: "Cleanup is still settling." } }; + expect(await taskRetryRefusal(held("cleanup_quarantined"), "company", "request", presentation)) + .toEqual({ reason: "cleanup_quarantined", message: "Cleanup is still settling." }); + expect(await taskRetryRefusal(held("cleanup_quarantined"), "company", "request", + { ...presentation, canRetry: true })).toBeNull(); + }); + + it("keeps a wait that the system finishes by itself accepted even when the task view hides Retry", async () => { + expect(await taskRetryRefusal(held("finalizing_saved_result"), "company", "request", + { canRetry: false, retryWait: { reason: "finalizing_saved_result", message: "Finishing." } })).toBeNull(); + }); + + it.each([ + ["a wait that another click can change", held("execution_settling")], + ["an admitted request", dbWithWake({ status: "coalesced", runId: "run", reason: "previous_execution_unverified" })], + ["a request without a recorded wait", dbWithWake({ status: "deferred_issue_execution", runId: null, reason: null })], + ["a missing request", dbWithWake(null)], + ])("reports no refusal for %s", async (_label, db) => { + expect(await taskRetryRefusal(db, "company", "request")).toBeNull(); + }); +}); diff --git a/server/src/services/workspace-recovery-preparation-dispatch.test.ts b/server/src/services/workspace-recovery-preparation-dispatch.test.ts index dd7121d8929..883c225319d 100644 --- a/server/src/services/workspace-recovery-preparation-dispatch.test.ts +++ b/server/src/services/workspace-recovery-preparation-dispatch.test.ts @@ -238,7 +238,14 @@ describe("automatic retained workspace recovery", () => { recoveryWorker = registerWorkspaceRecoveryWork(db, recoveryCoordinator, runtime, heartbeat.resumeRemoteStopComments); await recoveryWorker.ready; const retryIntent = { observedRunId: completed.id, idempotencyKey: randomUUID() }; - const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent).expect(202)).body; + // Until saved-work recovery verifies the files, admission refuses this + // Retry and the answer says so. The saved request still starts by + // itself once the files are verified. + const answer = await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent); + expect([200, 202]).toContain(answer.status); + if (answer.status === 200) expect(answer.body).toMatchObject({ accepted: false, + refused: { reason: "previous_execution_unverified" } }); + const retry = answer.body; const resumed = () => db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${retry.requestId}`)); await expect.poll(async () => await hasVerifiedWorkspaceRepair(db, await run(completed.id)), { timeout: 20_000 }).toBe(true); diff --git a/server/src/services/workspace-recovery-preparation-gates.test.ts b/server/src/services/workspace-recovery-preparation-gates.test.ts index 75de3f8dc10..efc7e2a704e 100644 --- a/server/src/services/workspace-recovery-preparation-gates.test.ts +++ b/server/src/services/workspace-recovery-preparation-gates.test.ts @@ -428,8 +428,9 @@ describe("automatic retained workspace recovery", () => { expect(refused).toMatchObject({ state: "exhausted", attempts: 1, failure: { code: "allocation_ownership_ambiguous", retryClass: "service_required" } }); await heartbeatService(db).resumeTaskRetry(f.companyId, first.requestId!); + // The route admitted this Retry once before answering (it waited then), + // so its earlier wait stays on the row as history; the request is admitted now. const [admitted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, first.requestId!)); - expect(admitted.payload?.executionWait).toBeUndefined(); expect(admitted).toMatchObject({ status: "coalesced", runId: expect.any(String), payload: { taskRetry: { strategy: "fresh_isolated_workspace" } } }); expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ generationId: refused!.generationId, @@ -635,9 +636,15 @@ describe("automatic retained workspace recovery", () => { : { latestRunId: later.id, latestRunStatus: "failed", latestRunErrorCode: "setup_failed" } }); await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: before.agentId, status: "running" }); const app = boardRecoveryApp(), retryIntent = { observedRunId: later.id, idempotencyKey: randomUUID() }; - const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent).expect(202)).body; - retryRequestId = retry.requestId; - expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent).expect(202)).body.requestId).toBe(retryRequestId); + // When admission refuses this Retry for a reason another click cannot + // change, the answer names the refusal instead of "accepted". A replay + // of the same click names the same saved request. + const first = await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent); + expect([200, 202]).toContain(first.status); + expect(first.body).toMatchObject(first.status === 200 + ? { accepted: false, refused: { reason: expect.any(String) } } : { accepted: true }); + retryRequestId = first.body.requestId; + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(retryIntent)).body.requestId).toBe(retryRequestId); await heartbeatService(db).resumeTaskRetry(f.companyId, retryRequestId!); const [registered] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId), sql`${issueRecoveryActions.evidence}->>'runId' = ${f.runId}`)); diff --git a/server/src/services/workspace-recovery-preparation-proof.test.ts b/server/src/services/workspace-recovery-preparation-proof.test.ts index 7dde4245b1e..2a58429de68 100644 --- a/server/src/services/workspace-recovery-preparation-proof.test.ts +++ b/server/src/services/workspace-recovery-preparation-proof.test.ts @@ -144,8 +144,10 @@ describe("automatic retained workspace recovery", () => { await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: f.runId, issueId: f.issueId, provider: "local", providerLeaseId: state === "provider_handle" ? randomUUID() : null, status: state === "active" ? "active" : "released", releasedAt: new Date(), cleanupStatus: "success" }); + // The answer names the refusal instead of reporting an accepted Retry. const receipt = (await request(boardRecoveryApp()).post(`/api/issues/${f.issueId}/retry`) - .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; + .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(200)).body; + expect(receipt).toMatchObject({ accepted: false, refused: { reason: expect.any(String), message: expect.any(String) } }); await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`))).toHaveLength(0); diff --git a/server/src/services/workspace-recovery-preparation.test.ts b/server/src/services/workspace-recovery-preparation.test.ts index 2f7047f37a8..f9c16a92981 100644 --- a/server/src/services/workspace-recovery-preparation.test.ts +++ b/server/src/services/workspace-recovery-preparation.test.ts @@ -717,8 +717,13 @@ describe("automatic retained workspace recovery", () => { : gate === "output_stream" ? { lastOutputStream: "stdout" } : gate === "compressed" ? { logCompressed: true } : {}; if (Object.keys(patch).length) await db.update(heartbeatRuns).set(patch).where(eq(heartbeatRuns.id, f.source.id)); - const receipt = (await request(boardRecoveryApp()).post(`/api/issues/${f.issueId}/retry`) - .send({ observedRunId: f.failed.id, idempotencyKey: randomUUID() }).expect(202)).body; + // Admission refuses this held Retry; the answer names the refusal + // instead of "accepted" when admission settles before the answer. + const answer = await request(boardRecoveryApp()).post(`/api/issues/${f.issueId}/retry`) + .send({ observedRunId: f.failed.id, idempotencyKey: randomUUID() }); + expect([200, 202]).toContain(answer.status); + if (answer.status === 200) expect(answer.body).toMatchObject({ accepted: false, refused: { reason: expect.any(String) } }); + const receipt = answer.body; await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`))).toHaveLength(0); @@ -877,9 +882,13 @@ describe("automatic retained workspace recovery", () => { try { await worker.ready; const app = boardRecoveryApp(), intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; - const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; - requestId = receipt.requestId; - expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(requestId); + // Until the stopped sandbox is confirmed, admission refuses this Retry + // and the answer names the refusal. The saved request starts later. + const answer = await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent); + expect([200, 202]).toContain(answer.status); + if (answer.status === 200) expect(answer.body).toMatchObject({ accepted: false, refused: { reason: expect.any(String) } }); + requestId = answer.body.requestId; + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent)).body.requestId).toBe(requestId); await expect.poll(() => f.runtime.retryPendingSandboxTeardown.mock.calls.length, { timeout: 20_000 }).toBe(1); expect(await successors()).toHaveLength(0); expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index 150da3e578d..5b53903e9e4 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -448,6 +448,7 @@ export const issuesApi = { api.post<{ runId: string; resultId: string; leaseId: string; status: "queued" }>(`/issues/${id}/recovery-actions/retry-workspace-export`, data), retryTask: (issueId: string, body: { idempotencyKey: string; observedRunId: string | null; retryAction?: "move_forward" }) => api.post<{ requestId: string | null; accepted: boolean; stale: boolean; alreadyMovedOn?: boolean; + refused?: { reason: string; message: string }; scheduledRetry?: { outcome: "promoted" | "already_promoted" | "gate_suppressed" | "waiting" | "no_scheduled_retry"; message: string }; presentation: import("@paperclipai/shared").TaskInterruptionPresentation | null }>(`/issues/${issueId}/retry`, body), recoverWorkspace: (id: string, data: { actionId: string; runId: string; idempotencyKey: string }) => diff --git a/ui/src/hooks/useTaskRetry.test.tsx b/ui/src/hooks/useTaskRetry.test.tsx index 4eb14cd3045..f7a1e603a57 100644 --- a/ui/src/hooks/useTaskRetry.test.tsx +++ b/ui/src/hooks/useTaskRetry.test.tsx @@ -41,6 +41,9 @@ describe("optimistic task Retry", () => { }); it.each([ ["a replaced attempt", { requestId: null, accepted: false, stale: true }, "already replaced, so Retry did not start new work"], + ["a Retry that admission refused", { requestId: "retry", accepted: false, stale: false, + refused: { reason: "previous_execution_unverified", message: "Retry can't continue because Paperclip can't confirm that the previous attempt stopped before it started work." } }, + "can't confirm that the previous attempt stopped"], ["a scheduled retry held by a gate", { requestId: "retry", accepted: true, stale: false, scheduledRetry: { outcome: "gate_suppressed" as const, message: "The company budget is paused." } }, "The company budget is paused."], ["a scheduled retry still waiting", { requestId: "retry", accepted: true, stale: false, diff --git a/ui/src/hooks/useTaskRetry.ts b/ui/src/hooks/useTaskRetry.ts index bbcb06fa6a2..497bd476e7e 100644 --- a/ui/src/hooks/useTaskRetry.ts +++ b/ui/src/hooks/useTaskRetry.ts @@ -15,6 +15,7 @@ export const STALE_TASK_RETRY_MESSAGE = "This attempt was already replaced, so R export const ALREADY_MOVED_ON_MESSAGE = "The task already moved on, so nothing new was started. It now shows its current state."; export function taskRetryNotStartedReason(result: RetryResult): string | null { + if (!result.accepted && result.refused) return result.refused.message; if (!result.accepted && result.alreadyMovedOn) return ALREADY_MOVED_ON_MESSAGE; if (!result.accepted) return STALE_TASK_RETRY_MESSAGE; const outcome = result.scheduledRetry?.outcome;