diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0716abc09d3..039cd6b4636 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1615,6 +1615,8 @@ timeout retries still require a fresh allocation under the policy above. A person's task Retry on a connected task first asks the chat retry owner to repeat the exact failed chat request. When the latest run is that person's own saved board turn that cannot be certified for replay, when no chat retry owner is available, or when that owner refuses (`chat_failed_run_retry_not_authorized`), the refusal and any writes it made are rolled back and the same Retry receipt becomes an ordinary board continuation: the clicking person is the responsible user, the run starts a fresh session from saved work, and no chat action, chat authority, or output replay is created. This is the same authority as a new message from that person. When the conversation still holds saved or unadmitted chat input for the task, Retry keeps the exact-replay path and its refusal, because that input continues through its own admission. Dispatch keeps every normal gate (current task access, executor containment, recovery holds, unadmitted chat input). Projection and the route therefore agree: a shown Retry on a connected task is accepted. Retry that Paperclip requests by itself on a person's behalf (restart recovery, provider limit reset) keeps the exact-replay requirement and still settles on the refusal. A budget resume continues the person's own Retry, so it takes the board continuation too. No migration is needed; tasks blocked by an earlier release become retryable at the next click. +The exact replay can also be refused after the click is accepted. This happens when chat source authorization was deferred by a busy identity or endpoint lock, or when it later fails, so the chat retry action settles `failed` with no admitted run. The task Retry receipt is then not failed. If it is a person's Retry, its source run is still the latest run, its chat wake has no run, and no saved or unadmitted chat input remains, the same receipt drops its chat action and becomes the board continuation described above, recorded as `issue.retry_continued_from_saved_work`. The deferred-receipt worker dispatches it with every normal gate. Retry that Paperclip requests by itself still settles as failed. When a failed run's saved work is recovered, the task's execution blocker offers Retry (also on a connected task) and says "Saved work recovered. Retry continues from it." A stopped (cancelled) run keeps "Send a message to continue." + ### Saved chat Retry preparation Task Retry can meet a busy chat identity or endpoint lock before its source is authorized. diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index dd651dcacba..3449138c448 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -179,6 +179,9 @@ import { statusDecisionEffects, environmentLeases, environments, + executionWorkspaces, + executionWorkspaceSnapshots, + workspaceOperations, principalPermissionGrants, projects, projectWorkspaces, @@ -65889,6 +65892,221 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); + it.each(["person", "person_paused_after_accept", "restart_recovery", "restart_recovery_joined_by_person"] as const)("continues an accepted GitHub task Retry from recovered saved work when its chat replay is refused after the click: %s", async initiator => { + const f = await currentGitHubTaskRetryFixture(false); + try { + const companyId = f.fixture.companyId, issueId = f.issue.id; + const leaseId = randomUUID(), providerLeaseId = randomUUID(), workspaceId = randomUUID(), snapshotId = randomUUID(); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId)); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, + name: "Review workspace", providerType: "cloud_sandbox", mode: "isolated_workspace", strategyType: "project_primary", + cwd: `/workspaces/${workspaceId}`, metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 2, + snapshotId, source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }); + await db.insert(executionWorkspaceSnapshots).values({ id: snapshotId, companyId, executionWorkspaceId: workspaceId, + sourceOperationId: randomUUID(), sourceRunId: f.runId, generation: 1, epoch: 2, status: "committed", sha256: "c".repeat(64) }); + await db.update(heartbeatRuns).set({ errorCode: "native_continuation_requires_reconciliation", + error: "runner_remote_process_identity_unavailable", + resultJson: { stopReason: "adapter_failed", workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { + schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } }, + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", epoch: 2, state: "saved", + leaseId, providerLeaseId, remoteCwd: `/workspaces/${workspaceId}`, snapshotId, workspaceId } }, + }).where(eq(heartbeatRuns.id, f.runId)); + const lease = { id: leaseId, providerLeaseId, companyId, issueId, heartbeatRunId: f.runId, provider: "daytona", + status: "expired", cleanupStatus: "success", releasedAt: new Date(), leasePolicy: "ephemeral" }; + await db.insert(environmentLeases).values({ ...lease, metadata: { remoteExecutionTermination: + remoteTerminationReceipt(lease, { providerLeaseId, state: "destroyed" }) } }); + const recoveryActionId = randomUUID(); + await db.insert(issueRecoveryActions).values({ id: recoveryActionId, companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", status: "resolved", ownerType: "system", outcome: "blocked", + cause: "native_continuation_requires_reconciliation", fingerprint: `native-workspace:${f.runId}`, + nextAction: "The original sandbox is retained for workspace repair.", evidence: { runId: f.runId, + automaticRecovery: { runId: f.runId, policy: "preserve_without_replay_v1", replay: "blocked", actionOutcome: "unknown" }, + systemRecovery: { ...createSystemRecovery(f.runId), state: "complete", phase: "reconcile_effects", attempts: 1, + preservationOperationIds: [randomUUID()] }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: "recovered", + stage: "saving_receipts", attempt: 1, retryable: true, requestIds: [], attemptLimit: 6 }, + continuationDelivery: "invalidated", workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] }, + } }); + await db.insert(workspaceOperations).values({ companyId, issueId, heartbeatRunId: f.runId, + executionWorkspaceId: workspaceId, phase: "workspace_recovery", status: "succeeded", finishedAt: new Date(), + metadata: { recoveryActionId, leaseId, providerLeaseId, snapshotId, generation: 1, sha256: "c".repeat(64), + storage: "remote_workspace", epoch: 2 } }); + const { getExecutionBlocker } = await import("../services/execution-blocker.js"); + const blocker = await getExecutionBlocker(db, companyId, issueId); + const interruption = await taskInterruptionPresentation(db, f.issue); + expect(interruption).toMatchObject({ state: "failed", canRetry: true }); + expect(blocker).toMatchObject({ recoveryActionId, canRetry: true, + nextAction: "Saved work recovered. Retry continues from it.", + workspaceRecovery: { status: "recovered", message: "Saved work recovered. Retry continues from it." } }); + expect(JSON.stringify(blocker)).not.toContain("Send a message"); + await db.update(heartbeatRuns).set({ status: "timed_out" }).where(eq(heartbeatRuns.id, f.runId)); + expect(await getExecutionBlocker(db, companyId, issueId)).toMatchObject({ canRetry: true, + nextAction: "Saved work recovered. Retry continues from it.", + workspaceRecovery: { message: "Saved work recovered. Retry continues from it." } }); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "native_session_cleanup_quarantined" }) + .where(eq(heartbeatRuns.id, f.runId)); + const quarantined = await getExecutionBlocker(db, companyId, issueId); + expect(quarantined).toMatchObject({ canRetry: false, nextAction: "Saved work recovered.", + workspaceRecovery: { message: "Saved work recovered." } }); + await db.update(heartbeatRuns).set({ errorCode: "native_continuation_requires_reconciliation" }) + .where(eq(heartbeatRuns.id, f.runId)); + + const identityLock = `chat-identity:${companyId}:${f.action.principalId}`; + const holdLock = async (lock: (tx: Parameters[0]>[0]) => Promise) => { + let release = () => {}, locked = () => {}; + const held = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { locked = resolve; }); + const holder = db.transaction(async tx => { await lock(tx); locked(); await held; }); + await ready; + return async () => { release(); await holder; }; + }; + const releaseIdentity = await holdLock(tx => tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${identityLock}, 0))`)); + let request: Awaited>; + try { + request = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry, + ...(initiator.startsWith("person") ? {} : { initiatedBy: "restart_recovery" as const }) }); + } finally { await releaseIdentity(); } + if (initiator === "restart_recovery_joined_by_person") { + const joined = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); + expect(joined).toMatchObject({ accepted: true, requestId: request.requestId, chatActionId: request.chatActionId }); + } + expect(request).toMatchObject({ accepted: true, stale: false, chatActionId: expect.any(String) }); + const chatAction = async () => (await db.select().from(chatActions).where(eq(chatActions.id, request.chatActionId!)))[0]; + expect((await chatAction()).payload.sourceAuthorizationPending).toBe(true); + + const [{ status: agentStatus }] = await db.select({ status: agents.status }).from(agents) + .where(eq(agents.id, f.fixture.assignedAgentId)); + if (initiator === "person_paused_after_accept") await db.update(agents).set({ status: "paused" }) + .where(eq(agents.id, f.fixture.assignedAgentId)); + const releaseAgent = initiator === "person_paused_after_accept" ? async () => {} + : await holdLock(tx => tx.select().from(agents).where(eq(agents.id, f.fixture.assignedAgentId)).for("update")); + try { + await vi.waitFor(async () => { + await f.service.processFailedChatRunRetry(request.chatActionId!); + expect((await chatAction()).status).toBe("failed"); + }, { timeout: 30_000, interval: 250 }); + } finally { await releaseAgent(); } + expect(await readAgentWakeupRequestsById(request.chatActionId!)).toHaveLength(0); + + await reconcileTaskChatRetryReceipts(db, { companyId, settleOnly: true }); + if (initiator === "restart_recovery") { + expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "failed", runId: null }); + expect((await readHeartbeatRunsByCompanyId(companyId)) + .filter(candidate => candidate.contextSnapshot?.taskRetryRequestId === request.requestId)).toHaveLength(0); + return; + } + const [continued] = await readAgentWakeupRequestsById(request.requestId!); + expect(continued).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(taskRetryModule.readTaskRetryIntent(continued.payload)?.chatActionId).toBeUndefined(); + if (initiator === "person_paused_after_accept") { + await expect(heartbeatService(db).resumeTaskRetry(companyId, request.requestId!)).rejects.toThrow("not invokable"); + expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect((await readHeartbeatRunsByCompanyId(companyId)) + .filter(candidate => candidate.contextSnapshot?.taskRetryRequestId === request.requestId)).toHaveLength(0); + await db.update(agents).set({ status: agentStatus }).where(eq(agents.id, f.fixture.assignedAgentId)); + } + await heartbeatService(db).resumeTaskRetry(companyId, request.requestId!); + const [admitted] = await readAgentWakeupRequestsById(request.requestId!); + expect(admitted).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + const [run] = await readHeartbeatRunsById(admitted.runId!); + expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId, taskRetryRequestId: request.requestId } }); + expect(await readFreshWorkspaceReadiness(db, run)).toMatchObject({ mode: "resume_preserved_workspace" }); + expect((await readHeartbeatRunsByCompanyId(companyId)) + .filter(candidate => candidate.contextSnapshot?.taskRetryRequestId === request.requestId)).toHaveLength(1); + const activity = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, "issue.retry_continued_from_saved_work"))); + expect(activity).toHaveLength(1); + expect(activity[0].details).toMatchObject({ requestId: request.requestId, refusedChatActionId: request.chatActionId }); + } finally { await f.service.shutdown(); } + }, 90_000); + + it.each(["unrecovered_workspace", "access_revoked", "paused_agent"] as const)( + "settles a refused GitHub task Retry by its own gates: %s", async gate => { + const f = await currentGitHubTaskRetryFixture(false); + try { + const companyId = f.fixture.companyId, issueId = f.issue.id; + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", ownerType: "system", outcome: "blocked", cause: "native_continuation_requires_reconciliation", + fingerprint: `native-workspace:${f.runId}`, nextAction: "Recovered", evidence: { runId: f.runId, + automaticRecovery: { runId: f.runId, policy: "preserve_without_replay_v1", replay: "blocked", actionOutcome: "unknown" }, + systemRecovery: { ...createSystemRecovery(f.runId), state: gate === "unrecovered_workspace" ? "running" : "complete", + phase: gate === "unrecovered_workspace" ? "preserve_files" : "reconcile_effects", attempts: 1, + preservationOperationIds: [randomUUID()] }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, + status: gate === "unrecovered_workspace" ? "recovering" : "recovered", + stage: gate === "unrecovered_workspace" ? "restoring_files" : "saving_receipts", attempt: 1, requestIds: [] } } }); + const request = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); + const [chat] = await db.update(chatActions).set({ status: "failed", + result: { code: "failed_chat_retry_not_authorized", attemptCount: 1 } }) + .where(eq(chatActions.id, request.chatActionId!)).returning(); + const [{ status: agentStatus }] = await db.select({ status: agents.status }).from(agents) + .where(eq(agents.id, f.fixture.assignedAgentId)); + if (gate === "access_revoked") await db.update(companyMemberships).set({ status: "suspended", updatedAt: new Date() }) + .where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, "owner-user"))); + if (gate === "paused_agent") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId)); + await taskRetryModule.settleRefusedTaskChatRetry(db, chat); + const retryRuns = async () => (await readHeartbeatRunsByCompanyId(companyId)) + .filter(candidate => candidate.contextSnapshot?.taskRetryRequestId === request.requestId); + if (gate !== "paused_agent") { + expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "failed", runId: null }); + expect(await retryRuns()).toHaveLength(0); + return; + } + expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "deferred_issue_execution", runId: null }); + await expect(heartbeatService(db).resumeTaskRetry(companyId, request.requestId!)).rejects.toThrow("not invokable"); + expect(await retryRuns()).toHaveLength(0); + await db.update(agents).set({ status: agentStatus }).where(eq(agents.id, f.fixture.assignedAgentId)); + await heartbeatService(db).resumeTaskRetry(companyId, request.requestId!); + expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + expect(await retryRuns()).toHaveLength(1); + } finally { await f.service.shutdown(); } + }, 60_000); + + it("keeps a refused GitHub task Retry continued when reconciliation read it before the refusal settled", async () => { + const f = await currentGitHubTaskRetryFixture(false); + try { + const companyId = f.fixture.companyId, issueId = f.issue.id; + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", ownerType: "system", outcome: "blocked", cause: "native_continuation_requires_reconciliation", + fingerprint: `native-workspace:${f.runId}`, nextAction: "Recovered", evidence: { runId: f.runId, + automaticRecovery: { runId: f.runId, policy: "preserve_without_replay_v1", replay: "blocked", actionOutcome: "unknown" }, + systemRecovery: { ...createSystemRecovery(f.runId), state: "complete", phase: "reconcile_effects", attempts: 1, + preservationOperationIds: [randomUUID()] }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: "recovered", + stage: "saving_receipts", attempt: 1, requestIds: [] } } }); + const request = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); + const [chat] = await db.update(chatActions).set({ status: "failed", + result: { code: "failed_chat_retry_not_authorized", attemptCount: 8 } }) + .where(eq(chatActions.id, request.chatActionId!)).returning(); + let release = () => {}, locked = () => {}; + const held = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { locked = resolve; }); + const settling = db.transaction(async tx => { + await tx.select().from(issues).where(eq(issues.id, issueId)).for("update"); + locked(); await held; + await taskRetryModule.settleRefusedTaskChatRetry(tx as unknown as Db, chat); + }); + await ready; + const reconciling = reconcileTaskChatRetryReceipts(db, { companyId, settleOnly: true }); + onTestFinished(() => release()); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select 1 from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 10_000, interval: 50 }); + release(); + await Promise.all([settling, reconciling]); + const [wake] = await readAgentWakeupRequestsById(request.requestId!); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(taskRetryModule.readTaskRetryIntent(wake.payload)?.chatActionId).toBeUndefined(); + } finally { await f.service.shutdown(); } + }, 60_000); + it.each(["legacy_receipt", "concurrent_clicks", "lost_dispatch_reply", "restart", "retry_failed_successor", "http", "handoff_rollback", "board_permission_wait", "peer_conversation", "backlog_51", "consumed_backlog_52"] as const)( "preserves one current GitHub Retry across %s", async mode => { const f = await currentGitHubTaskRetryFixture(true, mode.includes("backlog") ? 51 : mode === "peer_conversation" ? 2 : 1); diff --git a/server/src/services/chat-channels/task-retry-input.ts b/server/src/services/chat-channels/task-retry-input.ts index 8257a8ec079..18c0798d915 100644 --- a/server/src/services/chat-channels/task-retry-input.ts +++ b/server/src/services/chat-channels/task-retry-input.ts @@ -98,11 +98,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"), isNull(agentWakeupRequests.runId), sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${input.retryActionId}`)); - await tx.update(agentWakeupRequests).set({ status: "failed", finishedAt: new Date(), updatedAt: new Date() }).where(and( - eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, action.payload.taskRetryIntentId), - eq(agentWakeupRequests.reason, "task_retry"), isNull(agentWakeupRequests.runId), - inArray(agentWakeupRequests.status, ["deferred_issue_execution", "queued", "claimed"]), - sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' = ${action.id}`, - )); + const { settleRefusedTaskChatRetry } = await import("../task-retry.js"); + await settleRefusedTaskChatRetry(tx as unknown as Db, action); }); } diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index 379a76d7a82..f8cbe5453a7 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -14,7 +14,7 @@ import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA, nativeR import { unreconciledNativeExportPredicate } from "./native-runtime/native-workspace-reconstruction.js"; import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; -import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; +import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, recoveredWorkMessage, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; import { nativeManualAdmissionHoldsExecution } from "./native-runtime/native-runner-ownership.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; @@ -369,7 +369,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st )).limit(1) : []; const withoutWorkspace = Boolean(run?.driverKind === "grok_bot_mcp" && nativeRunHasNoWorkspaceAccess(run)); const workspaceRepairRequired = Boolean(restoreHold && !withoutWorkspace && (!run || !await hasVerifiedWorkspaceRepair(db, run))); - const workspaceRecovery = workspaceRecoveryDisplay(action.evidence, runId); + const workspaceRecoveryState = workspaceRecoveryDisplay(action.evidence, runId); const reconstructableExport = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"].includes(action.cause); const exportSource = reconstructableExport ? readNativeWorkspaceSyncReference(run?.runnerProfileJson?.nativeWorkspaceSync) : null; const [exportLease] = exportSource ? await db.select().from(environmentLeases).where(and( @@ -411,8 +411,8 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st // hold to protect its files. Offer the same Retry that admission accepts, so // an infrastructure failure (such as a restart) is not left to a new message. // An intentional Stop keeps "send a new message" as its only continuation. - const recoveredFailure = Boolean(restoreHold && run?.status === "failed" && !workspaceRepairRequired && - workspaceRecovery?.status === "recovered"); + const recoveredFailure = Boolean(restoreHold && (run?.status === "failed" || run?.status === "timed_out") && !workspaceRepairRequired && + workspaceRecoveryState?.status === "recovered"); const canRetry = Boolean((!restoreHold || recoveredFailure) && run && conversation?.assigneeAgentId === run.agentId && !["done", "cancelled"].includes(conversation.status) && await canRetryStoppedRun(db, run)); const runError = run && canRetry && isPreDispatchReviewWait(run) @@ -435,6 +435,9 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st )).where(and( eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), )).limit(1) : []; + const retryOffered = canRetry && (!chatBinding || recoveredFailure); + const workspaceRecovery = workspaceRecoveryState?.status === "recovered" + ? { ...workspaceRecoveryState, message: recoveredWorkMessage(run?.status, retryOffered) } : workspaceRecoveryState; let nextAction = action.status === "active" && action.cause === EXECUTION_FINALIZATION_DEADLINE_EXCEEDED ? EXECUTION_FINALIZATION_DEADLINE_MESSAGE : action.nextAction; // Retry is a person's control on this task. Say so, so an agent reading this @@ -469,13 +472,12 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st // A stopped reviewer can differ from the task owner who receives the work back. agentId: run?.agentId ?? null, cause: action.cause, - nextAction: (workspaceRecovery?.status === "recovered" && canRetry && !chatBinding - ? "Saved work recovered. Retry or send a message to continue." : workspaceRecovery?.message) ?? (reconstructionPending ? "Continuation from saved sources is queued and waiting for execution checks." : nextAction), + nextAction: workspaceRecovery?.message ?? (reconstructionPending ? "Continuation from saved sources is queued and waiting for execution checks." : nextAction), ...(workspaceRecovery ? { workspaceRecovery } : {}), runStatus: run?.status ?? null, runError: runError?.slice(0, 1024) ?? null, canContinue: eligibleContinuation && !chatBinding, - canRetry: canRetry && !chatBinding, + canRetry: retryOffered, ...(restoreHold ? { workspaceRepairRequired, canRecordWorkspaceRepair: workspaceRepairRequired && hasRequiredWorkspaceRecovery(action.evidence) } : {}), savedMessageCount: saved.length, ...(canReconstructWorkspace ? { canReconstructWorkspace, workspaceRecoveryStatusVersion: conversation!.statusVersion } : {}), diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index c9265157946..5b144c51fcf 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -132,12 +132,18 @@ describe("execution truth projection", () => { const projected = projectExecution(source, coordinator({ phase: "terminal_failure" }), [], action, now); expect(projected).toMatchObject({ phase: "recovery_needed", label: "Stopped", cause: action.cause, - nextAction: status === "recovered" ? "Saved work recovered. Send a message to continue." + nextAction: status === "recovered" ? "Saved work recovered." : status === "blocked" ? "The saved source cannot be read safely." : "Recovering saved work…", permittedActions: ["inspect_run"], successorRunId: null, }); expect(source.status).toBe("failed"); expect(action.evidence.automaticRecovery.replay).toBe("blocked"); + if (status === "recovered") { + expect(projectExecution({ ...source, status: "timed_out" }, coordinator({ phase: "terminal_failure" }), [], action, now) + .nextAction).toBe("Saved work recovered."); + expect(projectExecution({ ...source, errorCode: "native_session_cleanup_quarantined" }, + coordinator({ phase: "terminal_failure" }), [], action, now).nextAction).toBe("Saved work recovered."); + } expect(projectExecution(source, undefined, [], { ...action, evidence: { ...action.evidence, workspaceRecovery: { ...action.evidence.workspaceRecovery, runId: "5f227d75-b314-4511-bf23-9c615277e69b" } }, diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 03b6cd3e6af..8eb8ad22eac 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -339,7 +339,7 @@ export function projectExecution( Object.assign(projection, { phase, label }); if (recoveryAction?.status === "resolved" && recoveryAction.evidence?.automaticRecovery) { projection.cause = recoveryAction.cause; - projection.nextAction = workspaceRecoveryDisplay(recoveryAction.evidence, run.id)?.message + projection.nextAction = workspaceRecoveryDisplay(recoveryAction.evidence, run.id, { status: run.status, retryOffered: false })?.message ?? recoveryAction.nextAction; const continuation = recoveryAction.evidence.explicitUserContinuation as { previousRunId?: unknown; runId?: unknown } | undefined; diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index a6119c48f1a..e10e666f764 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -3,9 +3,10 @@ import { notifyExternalConversationSettlement } from "./external-conversation-se import { writeExecutionControl } from "./execution-control-notifications.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { z } from "zod"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; import { randomUUID } from "node:crypto"; import { taskRetryRunAncestry } from "./recovery-run-ancestry.js"; -import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, issueComments, runIdentityContexts, issueRecoveryActions, type Db } from "@paperclipai/db"; import { conflict, forbidden, notFound, HttpError } from "../errors.js"; import { persistActivity } from "./activity-log.js"; @@ -55,6 +56,7 @@ const taskRetryIntentSchema = z.object({ actorId: z.string().min(1), actorSource: z.enum(["local_implicit", "session", "board_key", "cloud_tenant"]).default("session"), strategy: z.enum(["evaluate", "continue", "fresh_isolated_workspace", "finalize_result"]), chatActionId: z.string().uuid().optional(), canonicalIntentId: z.string().uuid().optional(), queuedChatRequestId: z.string().uuid().optional(), + initiatedBy: z.enum(["restart_recovery", "provider_limit_reset"]).optional(), boardMessageOrigin: z.object({ commentId: z.string().uuid(), identityContextId: z.string().uuid() }).optional(), }); export type TaskRetryIntent = z.infer; @@ -400,7 +402,8 @@ function admitTaskRetry(input: TaskRetryRequest) { actorSource: taskRetryIntentSchema.shape.actorSource.parse(input.actorSource ?? "session"), strategy: "evaluate", ...(active && activeIntent?.chatActionId ? { chatActionId: activeIntent.chatActionId } : {}), ...(active && activeIntent?.queuedChatRequestId ? { queuedChatRequestId: activeIntent.queuedChatRequestId } : {}), - ...(active ? { canonicalIntentId: active.id } : {}) }; + ...(active ? { canonicalIntentId: active.id } : {}), + ...(input.initiatedBy && !active ? { initiatedBy: input.initiatedBy } : {}) }; // Preserve external chat authority through its existing admission owner. // Generic board Retry must never manufacture a chat-inbound request. const [chat] = await tx.select({ id: chatConversations.id }).from(chatConversations) @@ -427,10 +430,8 @@ function admitTaskRetry(input: TaskRetryRequest) { const savedBoardTurn = Boolean(chat && !active && !busy && !boardMessageOrigin && !intent.queuedChatRequestId && (latest.contextSnapshot?.source === "issue.comment.reopen" || latest.contextSnapshot?.boardMessageRetry === true) && latest.responsibleUserId === input.actorId); - const hasUnadmittedChatInput = async () => (await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, issue.id), - eq(issues.companyId, issue.companyId), unadmittedChatWakeupCondition(issues.id, issues.companyId))).limit(1)).length > 0 || - Boolean((await selectTaskRetryInput(tx, { companyId: issue.companyId, issueId: issue.id, - agentId: issue.assigneeAgentId!, failedRunId: latest.id }))?.sourceActionIds.length); + const hasUnadmittedChatInput = () => taskHasUnadmittedChatInput(db, { companyId: issue.companyId, issueId: issue.id, + agentId: issue.assigneeAgentId!, failedRunId: latest.id }); if (savedBoardTurn && (input.initiatedBy || await hasUnadmittedChatInput())) { throw conflict("This saved message can't be repeated exactly as it was sent. Retry continues the task from its saved work instead.", { code: "chat_failed_run_retry_not_authorized", reason: "saved_board_message_not_replayable" }); @@ -502,6 +503,81 @@ function admitTaskRetry(input: TaskRetryRequest) { }, { companyId: input.companyId }); } +async function taskHasUnadmittedChatInput(db: Db, input: { companyId: string; issueId: string; agentId: string; failedRunId: string }) { + const [held] = await db.select({ id: issues.id }).from(issues).where(and(eq(issues.id, input.issueId), + eq(issues.companyId, input.companyId), unadmittedChatWakeupCondition(issues.id, issues.companyId))).limit(1); + return Boolean(held) || Boolean((await selectTaskRetryInput(db, input))?.sourceActionIds.length); +} + +async function continueRefusedTaskChatRetry(db: Db, wake: typeof agentWakeupRequests.$inferSelect, + intent: TaskRetryIntent, chat: typeof chatActions.$inferSelect) { + if (intent.canonicalIntentId || intent.queuedChatRequestId || intent.generationId !== wake.id || + wake.requestedByActorType !== "user" || wake.requestedByActorId !== intent.actorId || + !wake.idempotencyKey?.startsWith("task-retry:") || chat.status !== "failed" || + chat.payload.taskRetryIntentId !== wake.id) return false; + const issueId = String(wake.payload?.issueId); + return db.transaction(async tx => { + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, issueId), + eq(issues.companyId, wake.companyId))).for("update"); + if (!issue || issue.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(issue.status)) return false; + const [latest] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, wake.companyId), or(eq(heartbeatRuns.issueId, issueId), eq(heartbeatRuns.nativeIssueId, issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`))) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + if (latest?.id !== intent.sourceRunId) return false; + if (intent.initiatedBy) { + const [joined] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.agentId, wake.agentId), + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "coalesced"), + isNull(agentWakeupRequests.runId), eq(agentWakeupRequests.requestedByActorType, "user"), + eq(agentWakeupRequests.requestedByActorId, intent.actorId), + sql`${agentWakeupRequests.payload}#>>'{taskRetry,canonicalIntentId}' = ${wake.id}`, + sql`${agentWakeupRequests.payload}#>'{taskRetry,initiatedBy}' is null`)).limit(1); + if (!joined) return false; + } + const recoveries = await tx.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, wake.companyId), eq(issueRecoveryActions.sourceIssueId, issueId), + sql`${issueRecoveryActions.evidence}->>'runId' = ${intent.sourceRunId}`)); + if (!recoveries.some(({ evidence }) => readSystemRecovery(evidence)?.state === "complete" && + readWorkspaceRecovery(evidence)?.runId === intent.sourceRunId && + readWorkspaceRecovery(evidence)?.status === "recovered")) return false; + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: wake.companyId, issueId, + actorId: intent.actorId, actorSource: intent.actorSource })) return false; + const [chatReceipt] = await tx.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId }) + .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.id, chat.id))); + if (chatReceipt && (chatReceipt.runId || !["failed", "skipped", "cancelled"].includes(chatReceipt.status))) return false; + if (await taskHasUnadmittedChatInput(tx as unknown as Db, { companyId: wake.companyId, issueId, + agentId: wake.agentId, failedRunId: intent.sourceRunId })) return false; + const { chatActionId: _replaced, initiatedBy: _automatic, ...continuation } = intent; + await notifyDeferredExecutionWork(tx, { companyId: wake.companyId, requestId: wake.id }); + const [converted] = await tx.update(agentWakeupRequests).set({ updatedAt: new Date(), + payload: { ...wake.payload, taskRetry: continuation, refusedChatRetry: { chatActionId: chat.id, + code: typeof chat.result?.code === "string" ? chat.result.code : null } } }) + .where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.runId} is null`, + sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' = ${chat.id}`)) + .returning({ id: agentWakeupRequests.id }); + if (!converted) return false; + await persistActivity(tx as unknown as Db, { companyId: wake.companyId, actorType: "system", actorId: "task-retry", + action: "issue.retry_continued_from_saved_work", entityType: "issue", entityId: issueId, runId: intent.sourceRunId, + details: { requestId: wake.id, sourceRunId: intent.sourceRunId, refusedChatActionId: chat.id } }); + return true; + }); +} + +export async function settleRefusedTaskChatRetry(db: Db, chat: typeof chatActions.$inferSelect) { + const intentId = chat.payload.taskRetryIntentId; + if (typeof intentId !== "string") return; + const unadmitted = and(eq(agentWakeupRequests.companyId, chat.companyId), eq(agentWakeupRequests.id, intentId), + eq(agentWakeupRequests.reason, "task_retry"), isNull(agentWakeupRequests.runId), + inArray(agentWakeupRequests.status, ["deferred_issue_execution", "queued", "claimed"]), + sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' = ${chat.id}`); + const [wake] = await db.select().from(agentWakeupRequests).where(unadmitted); + const intent = readTaskRetryIntent(wake?.payload); + if (!wake || intent && await continueRefusedTaskChatRetry(db, wake, intent, chat)) return; + await db.update(agentWakeupRequests).set({ status: "failed", finishedAt: new Date(), updatedAt: new Date() }).where(unadmitted); +} + /** Chat owns admission and authorization. The task receipt follows that * durable outcome; it never launches a parallel generic board execution. */ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) { @@ -570,12 +646,16 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId? eq(chatActions.status, "processed"), eq(chatActions.updatedAt, chat.updatedAt))); }); } + if (!intent.queuedChatRequestId && chat && !receipt?.runId && await continueRefusedTaskChatRetry(db, wake, intent, chat)) continue; const failed = inputDenied || (chat && ["failed", "cancelled"].includes(chat.status)); const write = (writer: Pick) => writer.update(agentWakeupRequests).set({ ...(receipt?.runId ? { status: "coalesced", runId: receipt.runId, finishedAt: new Date() } : failed ? { status: "failed", finishedAt: new Date() } : {}), updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"))); + }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), + intent.queuedChatRequestId + ? sql`${agentWakeupRequests.payload}#>>'{taskRetry,queuedChatRequestId}' = ${intent.queuedChatRequestId}` + : sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' = ${intent.chatActionId!}`)); if (receipt?.runId || failed) await db.transaction(async tx => { await notifyStrandedAgentStatus(tx, { companyId: wake.companyId, agentId: wake.agentId }); await notifyExternalConversationSettlement(tx, { companyId: wake.companyId, diff --git a/server/src/services/workspace-recovery-receipt.ts b/server/src/services/workspace-recovery-receipt.ts index 12dbd51bc50..11910813161 100644 --- a/server/src/services/workspace-recovery-receipt.ts +++ b/server/src/services/workspace-recovery-receipt.ts @@ -54,13 +54,22 @@ export function completedWorkspaceRecoveryCondition(runId: string | AnyColumn | ); } -export function workspaceRecoveryDisplay(evidence: Record, runId: string | null): ExecutionBlocker["workspaceRecovery"] { +export const RECOVERED_WORK_RETRY_MESSAGE = "Saved work recovered. Retry continues from it."; + +export function recoveredWorkMessage(runStatus: string | null | undefined, retryOffered: boolean) { + if (retryOffered) return RECOVERED_WORK_RETRY_MESSAGE; + return runStatus === "failed" || runStatus === "timed_out" ? "Saved work recovered." + : "Saved work recovered. Send a message to continue."; +} + +export function workspaceRecoveryDisplay(evidence: Record, runId: string | null, + run?: { status: string | null; retryOffered: boolean }): ExecutionBlocker["workspaceRecovery"] { const state = readWorkspaceRecovery(evidence); if (!state || state.runId !== runId) return undefined; return { status: state.status, stage: state.stage, canRetry: state.status === "blocked" && state.retryable === true, message: state.status === "queued" || state.status === "recovering" ? "Recovering saved work…" - : state.status === "recovered" ? "Saved work recovered. Send a message to continue." + : state.status === "recovered" ? recoveredWorkMessage(run?.status, run?.retryOffered ?? false) : blockedWorkspaceRecoveryMessage(evidence) ?? state.message ?? "Recovery of this task's saved work has not finished.", }; } diff --git a/server/src/services/workspace-recovery-retry-copyback.test.ts b/server/src/services/workspace-recovery-retry-copyback.test.ts index def29ff74d5..3a05b657318 100644 --- a/server/src/services/workspace-recovery-retry-copyback.test.ts +++ b/server/src/services/workspace-recovery-retry-copyback.test.ts @@ -296,7 +296,7 @@ describe("automatic retained workspace recovery", () => { const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); expect(blocker).toMatchObject({ runId: second.runId, cause: "native_continuation_requires_reconciliation", canRetry: true, - nextAction: "Saved work recovered. Retry or send a message to continue." }); + nextAction: "Saved work recovered. Retry continues from it." }); expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ canRetry: true, observedRunId: second.runId }); return { ...f, ...second, service: secondService, issue }; } diff --git a/ui/src/lib/activity-format.ts b/ui/src/lib/activity-format.ts index 0a005fc1ee6..5aa4cfc5409 100644 --- a/ui/src/lib/activity-format.ts +++ b/ui/src/lib/activity-format.ts @@ -75,6 +75,7 @@ const ACTIVITY_ROW_VERBS: Record = { "issue.accepted_plan_decomposition_updated": "updated accepted-plan decomposition on", "issue.recovery_action_opened": "opened a recovery action on", "issue.workspace_recovered": "recovered saved work for", + "issue.retry_continued_from_saved_work": "continued from saved work on", "issue.workspace_recovery_requested": "requested workspace recovery for", "issue.recovery_action_resolved": "resolved the recovery action on", "issue.recovery_action_escalated": "escalated the recovery action on", @@ -178,6 +179,7 @@ const ISSUE_ACTIVITY_LABELS: Record = { "issue.attribution_spoof_rejected": "tried to choose its own responsible user", "issue.recovery_action_opened": "Opened a source-scoped recovery action", "issue.workspace_recovered": "Saved work recovered.", + "issue.retry_continued_from_saved_work": "Retry continued from saved work.", "issue.workspace_recovery_requested": "Requested workspace recovery", "issue.recovery_action_resolved": "Resolved the recovery action", "issue.recovery_action_escalated": "Escalated the recovery action",