diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 7124bc887f3..75ee5f55fe8 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1726,6 +1726,18 @@ Projectless runs use their run ID as a transient workspace placeholder. That placeholder cannot authorize managed workspace reuse; those successors keep the existing fresh-workspace preparation path while retaining the old directory. +The cleanup gate (`native_session_cleanup_quarantined`) never asks a person to +act. Release of the run's environment settles it: remote lease termination +receipts, or the local stop proof above, retire the quarantined owner. When the +run's own saved checkpoint records a completed provider turn (for example, a +same-run disposition recovery that the gate refused), the work is saved. If the +task waits on unresolved blocker tasks, the task view shows that wait and no +failure banner. Otherwise the missing task disposition stays visible with plain +wording. Every quarantined run keeps Retry hidden, because the retry route refuses +it; a new message continues once the previous session has stopped. These rules +read only `errorCode` and `runnerProfileJson.sessionCheckpoint`, so runs that +earlier releases quarantined are shown the same way without a migration. + A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed. Other legacy queued-run gate cancellations also stay out of unknown-action reconciliation when the terminal run has no retained execution authority, session, workspace recovery, resources, or execution events. The same proof removes the effect of an old false recovery hold only for its exact company, task, and run; the audit row remains. Started or conflicting work retains its hold. Review waits retain their guarded Retry path. New work still passes the current approval, dependency, budget, pause, and ownership gates. diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index e76ea62817a..2931aa8b1e5 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -211,7 +211,7 @@ export class NativeSessionCleanupQuarantinedError extends Error { constructor() { super( - "native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine.", + "native_session_cleanup_quarantined: the previous session is still shutting down; Paperclip finishes the cleanup automatically and the saved work is kept.", ); this.name = "NativeSessionCleanupQuarantinedError"; } diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.ts index 493f68a984f..bf511eb184f 100644 --- a/server/src/__tests__/agent-live-run-routes.test.ts +++ b/server/src/__tests__/agent-live-run-routes.test.ts @@ -1201,7 +1201,7 @@ describe("agent live run routes", () => { type: "board", userId: "operator", source: "session", companyIds: ["company-1"], }), url => request(url).post(`/api/agents/${routeAgentId}/wakeup`).send(retryBody)); expect(res.status).toBe(409); - expect(res.body).toMatchObject({ error: expect.stringContaining("cleanup and reconciliation") }); + expect(res.body).toMatchObject({ error: expect.stringContaining("still shutting down") }); expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled(); expect(mockChatRunRetries.prepareFailedChatRunRetry).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index fcdcf2dc530..2f658a6ef89 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -188,6 +188,46 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => expect((await taskInterruptionPresentation(db, task))?.retryWait?.reason).toBe("budget.blocked"); }); + // Shape saved by the previous release for a remote native run: the provider + // turn completed (checkpoint terminal), a same-run recovery was then refused by + // the cleanup gate. No migration rewrites it; the projection reads it as is. + it("reads a quarantine saved after a completed turn as settled work, not a failed attempt", async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), otherRunId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Completed turn", issuePrefix: `K${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", adapterType: "paperclip_runner" }); + const [task] = await db.insert(issues).values({ companyId, title: "Set up Storie", identifier: "CMP-7", + status: "blocked", assigneeAgentId: agentId }).returning(); + const staged = (id: string, turnTerminalState: string) => ({ id, companyId, agentId, nativeIssueId: task.id, scopeKind: "issue", + status: "failed", runtimeMode: "native", errorCode: "native_session_cleanup_quarantined", + error: "native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine.", + finishedAt: new Date(Date.now() - 60_000), + resultJson: { apiToolReceipts: { receipt: { state: "completed" } }, semanticToolReceipts: { doc: { state: "completed" } } }, + runnerProfileJson: { mode: "native", backend: "codex_app_server", sessionCheckpoint: { + cursor: "818", activeTurnId: null, semanticResult: null, driverKind: "codex_app_server", backendKind: "runner", + identity: { runId: id, agentId, issueId: task.id, companyId, sessionId: randomUUID() }, + terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState, + reportedWorkDisposition: "yielded" }, + terminalTurns: [{ turnId: randomUUID(), fingerprint: "{}" }], + } } }) as typeof heartbeatRuns.$inferInsert; + await db.insert(heartbeatRuns).values(staged(runId, "completed")); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: task.id, runId, phase: "terminal_failure", + attempt: 2, failureCode: "native_session_cleanup_quarantined" }); + // Without its own wait, a missing task disposition stays visible, with no Retry. + expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "failed", canRetry: false, + retryWait: { reason: "cleanup_quarantined" } }); + // The task waits on an active child: it shows that wait, not a failed attempt. + const [child] = await db.insert(issues).values({ companyId, title: "Verify review", identifier: "CMP-8", + status: "in_progress", assigneeAgentId: agentId }).returning(); + await db.insert(issueRelations).values({ companyId, issueId: child.id, relatedIssueId: task.id, type: "blocks" }); + expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "resolved", canRetry: false }); + // A quarantine whose turn did not complete is still unfinished, but it never + // offers the Retry that the retry route refuses. + await db.update(heartbeatRuns).set({ createdAt: new Date(Date.now() - 120_000) }).where(eq(heartbeatRuns.id, runId)); + await db.insert(heartbeatRuns).values(staged(otherRunId, "interrupted")); + expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "failed", canRetry: false, + observedRunId: otherRunId, retryWait: { reason: "cleanup_quarantined" } }); + }); + it("shows a queued-message Interrupt as handing off, not as a failed attempt", async () => { const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), userId = "board-user"; await db.insert(companies).values({ id: companyId, name: "Interrupted task", issuePrefix: `Q${companyId.slice(0, 7)}` }); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 055d152a447..ed8c42f935b 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -12,6 +12,7 @@ import { completeSetupTokenAiConnection } from "../services/setup-token-ai-conne import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js"; import { connectionManagementGuard, connectionRequestContext, connectionRequestUserId, aiLoginUserAction } from "./connection-management-authz.js"; import { cancellationRequestId } from "../services/native-runtime/native-cancellation-request.js"; +import { NATIVE_CLEANUP_SETTLING_MESSAGE } from "../services/native-runtime/completed-turn-cleanup.js"; import { agentHarnessType, isEnvironmentDriverSupportedForAdapter, normalizeClaudeRunnerOptions, type AgentConfigurationWarning, type AgentRunnerChoice } from "@paperclipai/shared"; import { resolveNewAgentRunnerForCompany } from "../services/agent-runner-selection.js"; import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED, pluginEnvironmentRunnerTaskDriver, pluginRunnerAiConnectionRequiredMessage, pluginRunnerAiConnectionRequiredTestResult } from "../services/plugin-runner-ai-connection.js"; @@ -6228,7 +6229,7 @@ export function agentRoutes( throw notFound("Failed run not found"); } if (failedRun.runtimeMode === "native" && failedRun.errorCode === "native_session_cleanup_quarantined") { - throw conflict("The stopped native session requires cleanup and reconciliation before a new attempt.", { + throw conflict(NATIVE_CLEANUP_SETTLING_MESSAGE, { code: "native_session_cleanup_quarantined", }); } diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index 3352d60d9b8..9509dfe9976 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -17,6 +17,7 @@ import { readNativeWorkspaceSyncReference } from "./native-runtime/native-worksp import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; import { nativeManualAdmissionHoldsExecution } from "./native-runtime/native-runner-ownership.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; +import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; const blockerContextKeys = ["issueId", "taskId", "source", "connectionIntentResolution", "interactionKind", "interactionStatus", "mutation", "interactionId"] as const; @@ -386,6 +387,9 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st !["done", "cancelled"].includes(conversation.status) && await canRetryStoppedRun(db, run)); const runError = run && canRetry && isPreDispatchReviewWait(run) ? "Waiting for review; this continuation never started." + // Earlier releases saved operator-facing text for the cleanup gate. Show + // the plain wording from the code instead of the stored message. + : run?.errorCode === NATIVE_SESSION_CLEANUP_QUARANTINED ? NATIVE_CLEANUP_SETTLING_MESSAGE : cancellation?.reason ?? (run?.status === "cancelled" ? "Execution was cancelled; its source was not recorded." // Rows written before SQL text was kept out of run errors still carry it. diff --git a/server/src/services/explicit-native-continuation.test.ts b/server/src/services/explicit-native-continuation.test.ts index 8c17a7772ba..5c61f923aa8 100644 --- a/server/src/services/explicit-native-continuation.test.ts +++ b/server/src/services/explicit-native-continuation.test.ts @@ -1188,6 +1188,19 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes expect(source.resultJson).toEqual(resultJson); }); + // Rows saved by earlier releases carry operator-facing text. The task + // notice shows plain wording instead; nothing asks an administrator to act. + it("shows plain wording for a cleanup refusal saved with the earlier operator text", async () => { + const f = await seed(); + await db.update(heartbeatRuns).set({ errorCode: "native_session_cleanup_quarantined", + error: "native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine." }) + .where(eq(heartbeatRuns.id, f.sourceRunId)); + const notice = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(notice).toMatchObject({ runId: f.sourceRunId, canRetry: false }); + expect(notice?.runError).toMatch(/still shutting down/); + expect(notice?.runError).not.toMatch(/operator|admin|controlled restart/i); + }); + it.each(["verified", "unproven", "changed", "dry_run", "retry", "duplicate"])( "requires exact local cleanup for a new turn after worker loss (%s)", async mode => { const f = await seed(); diff --git a/server/src/services/native-runtime/completed-turn-cleanup.ts b/server/src/services/native-runtime/completed-turn-cleanup.ts new file mode 100644 index 00000000000..3e599986e49 --- /dev/null +++ b/server/src/services/native-runtime/completed-turn-cleanup.ts @@ -0,0 +1,50 @@ +/** + * A native run can finish its provider turn and then be refused by the + * runner's cleanup gate: the previous session's shutdown is still unproven + * (`native_session_cleanup_quarantined`). The work of that turn is already + * saved. Cleanup settles on its own once the environment proves the old + * runner, provider and tools stopped (lease release and termination receipts, + * see `acknowledgeRemoteStop` in heartbeat/run-control.ts). It is not a failed + * attempt and it never needs an administrator. + * + * Every rule here reads only fields earlier releases already persisted + * (`errorCode` and `runnerProfileJson.sessionCheckpoint`), so runs quarantined + * before this change are classified the same way without a migration. + */ + +export const NATIVE_SESSION_CLEANUP_QUARANTINED = "native_session_cleanup_quarantined"; + +/** Plain wording for the cleanup gate. It names no person who must act. */ +export const NATIVE_CLEANUP_SETTLING_MESSAGE = + "The previous session is still shutting down. Your work is saved, and Paperclip finishes the cleanup automatically."; + +/** Shown for a quarantined run whose provider turn had already completed. */ +export const NATIVE_COMPLETED_TURN_CLEANUP_MESSAGE = + "This run finished its work and saved it. Paperclip finishes the session cleanup automatically. Send a message to continue; it starts once the previous session has stopped."; + +const record = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; + +/** The run's own saved checkpoint records a completed provider turn and no + * turn still active. Only the checkpoint saved for this exact run counts. */ +export function nativeCheckpointRecordsCompletedTurn(runnerProfileJson: unknown, runId: string): boolean { + const checkpoint = record(record(runnerProfileJson).sessionCheckpoint); + const terminal = record(checkpoint.terminal); + return record(checkpoint.identity).runId === runId + && (checkpoint.activeTurnId === null || checkpoint.activeTurnId === undefined) + && terminal.turnTerminalState === "completed"; +} + +/** A failed native run whose only failure is unproven cleanup after its + * provider turn completed. Its work is done; it is not an attempt to retry. */ +export function isCompletedTurnCleanupQuarantine(run: { + id: string; + runtimeMode?: string | null; + status?: string | null; + errorCode?: string | null; + runnerProfileJson?: unknown; +}): boolean { + return run.runtimeMode === "native" && run.status === "failed" + && run.errorCode === NATIVE_SESSION_CLEANUP_QUARANTINED + && nativeCheckpointRecordsCompletedTurn(run.runnerProfileJson, run.id); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 419a343ef2e..5bd1bcaa41a 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -13997,7 +13997,7 @@ describe("native session bounded recovery", () => { } }); - it("persists actionable operator recovery without an automatic cleanup wake", async () => { + it("persists plain cleanup recovery without an automatic cleanup wake", async () => { const updates: Array<{ table: unknown; values: Record }> = []; const failure = new NativeSessionCleanupQuarantinedError(); @@ -14028,7 +14028,7 @@ describe("native session bounded recovery", () => { nextAttemptAt: null, failureDetail: { nextAction: expect.stringContaining( - "Clearing a task session does not resolve this quarantine", + "it starts once the previous session has stopped", ), }, }); @@ -14039,7 +14039,7 @@ describe("native session bounded recovery", () => { ownerType: "board", wakePolicy: null, nextAction: expect.stringContaining( - "Clearing a task session does not resolve this quarantine", + "it starts once the previous session has stopped", ), }), ); @@ -14053,6 +14053,45 @@ describe("native session bounded recovery", () => { } }); + // Remote native run shape: the provider turn completed and its work was + // saved; a same-run recovery was then refused by the cleanup gate. The task + // disposition can still be missing, so the hold stays; the wording names the + // saved work and a message, never an administrator. + it("explains a cleanup refusal after a completed turn without asking an operator", async () => { + const updates: Array<{ table: unknown; values: Record }> = []; + const failure = new NativeSessionCleanupQuarantinedError(); + state.execute.mockReset().mockRejectedValueOnce(failure); + state.upsertRecoveryAction.mockReset().mockResolvedValue({}); + const updateIssue = vi.fn(async () => ({ status: "blocked", statusVersion: 7 })); + const service = vi.spyOn(issueServiceModule, "issueService") + .mockReturnValue({ update: updateIssue } as unknown as ReturnType); + const completedCheckpoint = { sessionCheckpoint: { + activeTurnId: null, semanticResult: null, cursor: "818", + identity: { runId: execution.binding.runId, agentId: execution.binding.agentId, + issueId: execution.binding.issueId, companyId: execution.binding.companyId, sessionId: "session" }, + terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", + turnTerminalState: "completed", reportedWorkDisposition: "yielded" }, + } }; + try { + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, {}, updates, completedCheckpoint), execution, runnerInstanceId: "runner", + })).rejects.toBe(failure); + const terminal = updates.find(entry => entry.table === nativeRunFinalizations + && entry.values.phase === "terminal_failure")?.values; + // Still a permanent, named refusal: no provider work is replayed. + expect(terminal).toMatchObject({ failureCode: "native_session_cleanup_quarantined", nextAttemptAt: null }); + const nextAction = String((terminal?.failureDetail as Record).nextAction); + expect(nextAction).toMatch(/finished its work and saved it/); + expect(nextAction).not.toMatch(/operator|admin|Clearing a task session/i); + expect(state.upsertRecoveryAction).toHaveBeenCalledWith(expect.objectContaining({ + cause: "native_session_cleanup_quarantined", wakePolicy: null, + nextAction: expect.stringMatching(/finished its work and saved it/), + })); + } finally { + service.mockRestore(); + } + }); + it.each(["persisted", "logging_failure", "recovery_write_failure"])( "signals an ownership hold instead of terminal teardown after authentication timeout (%s)", async (failureMode) => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 177501a0809..1249a491fe2 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9,6 +9,8 @@ import { nativePostResultFailureCategory, type NativePostResultFailureCategory } import { logger } from "../../middleware/logger.js"; import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js"; import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; +import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_COMPLETED_TURN_CLEANUP_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED, + nativeCheckpointRecordsCompletedTurn } from "./completed-turn-cleanup.js"; import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; @@ -8302,8 +8304,11 @@ const NATIVE_STARTUP_MODEL_REJECTED_ACTION = const NATIVE_TURN_MODEL_REJECTED_ACTION = "The provider rejected the selected model. Choose a supported model for this agent, then retry the task. Automatic retries are stopped."; -const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = - "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; +// The cleanup gate needs no person: environment release proves the old +// runner, provider and tools stopped, and the agent continues on its next wake. +const nativeCleanupRefusalAction = (completedTurnCleanup: boolean) => completedTurnCleanup + ? NATIVE_COMPLETED_TURN_CLEANUP_MESSAGE + : `${NATIVE_CLEANUP_SETTLING_MESSAGE} Send a message to continue; it starts once the previous session has stopped.`; const PROVIDER_DURABLE_EVENT_TYPES = NATIVE_PROVIDER_DURABLE_EVENT_TYPES; @@ -11918,6 +11923,15 @@ async function executePaperclipNativeSessionAttempt( : startupRetry ? startupRetry.nextAttemptAt : disposition.nextAttemptAt; + // A cleanup-gate refusal after this run's own turn completed (for example + // a same-run disposition recovery) leaves saved work; say so plainly. The + // task disposition may still be missing, so ownership is unchanged. + const completedTurnCleanup = sourceFailureCode === NATIVE_SESSION_CLEANUP_QUARANTINED + && phase === "terminal_failure" && await input.db + .select({ id: heartbeatRuns.id, runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, input.execution.binding.runId)) + .then(rows => rows.some(row => nativeCheckpointRecordsCompletedTurn(row.runnerProfileJson, input.execution.binding.runId))) + .catch(() => false); const recoveryProjection = nativeSessionRecoveryProjection({ phase, failureCode, @@ -11992,7 +12006,7 @@ async function executePaperclipNativeSessionAttempt( : sourceFailureCode === "native_provider_approval_required" ? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler." : sourceFailureCode === "native_session_cleanup_quarantined" - ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE + ? nativeCleanupRefusalAction(completedTurnCleanup) : sourceFailureCode === "runner_remote_provider_verification_unavailable" ? "Provider files could not be verified after two read attempts. Inspect the retained workspace and command transport before retrying the original run." : sourceFailureCode === "runner_ingress_access_rejected" @@ -12255,7 +12269,7 @@ async function executePaperclipNativeSessionAttempt( : sourceFailureCode === "native_provider_approval_required" ? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler." : sourceFailureCode === "native_session_cleanup_quarantined" - ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE + ? nativeCleanupRefusalAction(completedTurnCleanup) : sourceFailureCode === "runner_ingress_access_rejected" ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." : sourceFailureCode === "native_provider_terminal_failed" diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index c98c95f221c..93154e7e4d3 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -454,3 +454,39 @@ describe("setup command failure evidence", () => { for (let index = 0; index + 8 <= secret.length; index += 1) expect(text).not.toContain(secret.slice(index, index + 8)); }); }); + +// Remote native run shape: the provider turn completed and its work was +// saved, then a same-run recovery was refused by the cleanup gate. The task +// showed "Couldn't finish this attempt" with a Retry the server refuses (409). +describe("native cleanup quarantine presentation", () => { + const quarantined = { ...run, runtimeMode: "native" as const, errorCode: "native_session_cleanup_quarantined", + error: "native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine.", + resultJson: null, finishedAt: new Date("2026-11-10T14:39:31.873Z") }; + + it("shows the task's blocker wait, not a failed attempt, after a completed turn", () => { + expect(projectTaskInterruption(issue, { run: { ...quarantined, completedTurnCleanup: true }, agent, + waitingOnBlockers: true, paused: false, approval: false })).toMatchObject({ state: "resolved", canRetry: false }); + }); + + it("keeps a missing disposition visible when the task has no wait of its own", () => { + expect(projectTaskInterruption(issue, { run: { ...quarantined, completedTurnCleanup: true }, agent, + paused: false, approval: false })).toMatchObject({ state: "failed", canRetry: false, + retryWait: { reason: "cleanup_quarantined" } }); + }); + + it("never offers a Retry that the retry route refuses", () => { + const presentation = projectTaskInterruption(issue, { run: quarantined, agent, paused: false, approval: false }); + expect(presentation).toMatchObject({ state: "failed", canRetry: false, + retryWait: { reason: "cleanup_quarantined" } }); + expect(presentation?.retryWait?.message).not.toMatch(/operator|admin/i); + // Every other path that offers Retry is refused for the same run too. + const preparation = { id: "retry", requestedAt: new Date(0), sourceRunId: quarantined.id, preparationStatus: "failed" }; + expect(projectTaskInterruption(issue, { run: quarantined, agent, retryWait: preparation, paused: false, approval: false })) + .toMatchObject({ canRetry: false }); + }); + + it("keeps Retry for ordinary failed native attempts", () => { + expect(projectTaskInterruption(issue, { run: { ...quarantined, errorCode: "native_session_interrupted" }, agent, + paused: false, approval: false })).toMatchObject({ state: "failed", canRetry: true }); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 7d8fc3b82dd..2163074b6b1 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -19,6 +19,7 @@ import { redactCurrentUserText } from "../log-redaction.js"; import { logger } from "../middleware/logger.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; import { chatActionDeadline } from "./chat-action-work.js"; +import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; import { isQueuedDependencyWait } from "./run-cancellation.js"; import { AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON, WORKSPACE_BUSY_RETRY_REASON } from "../modules/run-dispatch/index.js"; @@ -46,6 +47,8 @@ type ProjectionRun = Pick = { saved_work_recovery_refused: "Paperclip refused to recover the saved files because a safety check on them cannot pass. " + "Retry runs the same check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work.", workspace_owner_stop_required: "Retry is waiting until Paperclip confirms that the previous server stopped saving files. The agent's result and your Retry are saved.", - cleanup_quarantined: "Retry cannot continue this attempt. Send a new message after the previous execution has stopped.", + cleanup_quarantined: NATIVE_CLEANUP_SETTLING_MESSAGE, cancelled_by_operator: "This attempt was stopped. Send a new message to continue from saved work.", // A new click replaces a Retry declined for its requester's access with a // fresh attempt under the clicker's access, so Retry stays available. @@ -172,8 +177,16 @@ function deniedProjectId(issue: ProjectionIssue, run?: ProjectionRun): string | } /** Presentation never grants authority. Dispatch repeats all gates under its - * locks. The same projection is used in detail, lists and attention surfaces. */ + * locks. The same projection is used in detail, lists and attention surfaces. + * It never offers a Retry that the retry route refuses: a native run refused + * by the cleanup gate cannot be retried (routes/agents.ts, canRetryStoppedRun). */ export function projectTaskInterruption(issue: ProjectionIssue, input: ProjectionState, now = Date.now()): TaskInterruptionPresentation | null { + const presentation = projectTaskInterruptionState(issue, input, now); + if (!presentation?.canRetry || input.run?.errorCode !== NATIVE_SESSION_CLEANUP_QUARANTINED) return presentation; + return { ...presentation, canRetry: false }; +} + +function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionState, now: number): TaskInterruptionPresentation | null { const { run, wait, action, agent, project, company } = input; const retryWait = input.retryWait ?? wait; if (!run) { @@ -300,6 +313,15 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio } if (wait && wait.requestedAt.getTime() > now - 10_000 && system?.state !== "exhausted") return { ...base, state: "working" }; + if (run.errorCode === NATIVE_SESSION_CLEANUP_QUARANTINED) { + // The provider turn completed and its work is saved; only shutdown of the + // old session is unproven, and that settles by itself. A task that waits on + // its blockers shows that wait, not a failed attempt. Without such a wait a + // missing task disposition stays visible. + if (run.completedTurnCleanup && input.waitingOnBlockers) return base; + return { ...base, state: "failed", canRetry: false, + retryWait: { reason: "cleanup_quarantined", message: NATIVE_CLEANUP_SETTLING_MESSAGE } }; + } return { ...base, state: "failed", canRetry: true, ...(projectAccess ? { projectAccess } : {}) }; } @@ -347,6 +369,12 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, // The model this run asked the provider for; configuration, not provider text. + // Saved fields only, so quarantines written by earlier releases read the same. + completedTurnCleanup: sql`coalesce(${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed' + and ${heartbeatRuns.errorCode} = ${NATIVE_SESSION_CLEANUP_QUARANTINED} + and ${heartbeatRuns.runnerProfileJson}#>>'{sessionCheckpoint,identity,runId}' = ${heartbeatRuns.id}::text + and ${heartbeatRuns.runnerProfileJson}#>>'{sessionCheckpoint,terminal,turnTerminalState}' = 'completed' + and coalesce(jsonb_typeof(${heartbeatRuns.runnerProfileJson}#>'{sessionCheckpoint,activeTurnId}'), 'null') = 'null', false)`, rejectedModel: sql`case when ${heartbeatRuns.errorCode} = 'native_provider_model_rejected' then ${boundedJsonString(sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'provider'->'model'`, 200)} end`, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, @@ -525,6 +553,16 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i savingFilesOnly: ids.every(id => saving.has(id)) }); } } + // A completed turn refused only by the cleanup gate shows the task's own + // blocker wait. Read readiness for those tasks only. + const settledIssueIds = [...runMap.values()].filter(run => run.completedTurnCleanup).map(run => run.issueId); + const waitingOnBlockers = new Set(); + if (settledIssueIds.length) { + const { issueService } = await import("./issues.js"); + const readiness = await issueService(db).listDependencyReadiness(companyId, settledIssueIds); + for (const [waitingIssueId, row] of readiness) + if (row.unresolvedBlockerIssueIds.length) waitingOnBlockers.add(waitingIssueId); + } for (const wait of currentWaits) { const current = waitMap.get(wait.issueId); if (!current || wait.requestedAt > current.requestedAt || @@ -586,6 +624,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i unidentifiedLaunchConfirmable: confirmableLaunches.has(issue.id), admissionWait: admissionWaitMap.get(issue.id), retryDependencies: retryDependencyMap.get(issue.id), + waitingOnBlockers: waitingOnBlockers.has(issue.id), }, now); result.set(issue.id, presentation); const run = facts.run;