diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 69abce42052..61d67e92312 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1624,7 +1624,7 @@ timeout retries still require a fresh allocation under the policy above. A person's task Retry on a connected task first asks the chat retry owner to repeat the exact failed chat request. When the latest run is that person's own saved board turn that cannot be certified for replay, when no chat retry owner is available, or when that owner refuses (`chat_failed_run_retry_not_authorized`), the refusal and any writes it made are rolled back and the same Retry receipt becomes an ordinary board continuation: the clicking person is the responsible user, the run starts a fresh session from saved work, and no chat action, chat authority, or output replay is created. This is the same authority as a new message from that person. When the conversation still holds saved or unadmitted chat input for the task, Retry keeps the exact-replay path and its refusal, because that input continues through its own admission. Dispatch keeps every normal gate (current task access, executor containment, recovery holds, unadmitted chat input). Projection and the route therefore agree: a shown Retry on a connected task is accepted. Retry that Paperclip requests by itself on a person's behalf (restart recovery, provider limit reset) keeps the exact-replay requirement and still settles on the refusal. A budget resume continues the person's own Retry, so it takes the board continuation too. No migration is needed; tasks blocked by an earlier release become retryable at the next click. -The exact replay can also be refused after the click is accepted. This happens when chat source authorization was deferred by a busy identity or endpoint lock, or when it later fails, so the chat retry action settles `failed` with no admitted run. The task Retry receipt is then not failed. If it is a person's Retry, its source run is still the latest run, its chat wake has no run, and no saved or unadmitted chat input remains, the same receipt drops its chat action and becomes the board continuation described above, recorded as `issue.retry_continued_from_saved_work`. The deferred-receipt worker dispatches it with every normal gate. Retry that Paperclip requests by itself still settles as failed. When a failed run's saved work is recovered, the task's execution blocker offers Retry (also on a connected task) and says "Saved work recovered. Retry continues from it." A stopped (cancelled) run keeps "Send a message to continue." +The exact replay can also be refused after the click is accepted. This happens when chat source authorization was deferred by a busy identity or endpoint lock, or when it later fails, so the chat retry action settles `failed` with no admitted run. The task Retry receipt is then not failed. If it is a person's Retry, its source run is still the latest run, its chat wake has no run, and no saved or unadmitted chat input remains, the same receipt drops its chat action and becomes the board continuation described above, recorded as `issue.retry_continued_from_saved_work`. Saved chat input whose source comment was written no later than the failed run's stop boundary (the user's recorded Stop request, otherwise `finishedAt`, the same boundary admission uses) does not block this: admission holds such input as `message_predates_stop`, so it can never start alone. The continuation adopts it (the input receipt is coalesced into the Retry receipt) and the task continues from saved work. Saved input that arrived after the stop keeps its own admission path, and the Retry still settles as failed. When the latest run is an admission that the recovery hold cancelled before dispatch (`execution_reconciliation_required`), the saved work that counts is the recovered work of the run whose recovery action held it. The deferred-receipt worker dispatches it with every normal gate. Retry that Paperclip requests by itself still settles as failed. When a failed run's saved work is recovered, the task's execution blocker offers Retry (also on a connected task) and says "Saved work recovered. Retry continues from it." A stopped (cancelled) run keeps "Send a message to continue." ### Saved chat Retry preparation diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 3449138c448..f364b7f048b 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -65892,11 +65892,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); - it.each(["person", "person_paused_after_accept", "restart_recovery", "restart_recovery_joined_by_person"] as const)("continues an accepted GitHub task Retry from recovered saved work when its chat replay is refused after the click: %s", async initiator => { - const f = await currentGitHubTaskRetryFixture(false); + it.each(["person", "person_with_input_before_stop", "person_after_held_admission", "person_paused_after_accept", "restart_recovery", "restart_recovery_joined_by_person"] as const)("continues an accepted GitHub task Retry from recovered saved work when its chat replay is refused after the click: %s", async initiator => { + const f = await currentGitHubTaskRetryFixture(initiator === "person_with_input_before_stop"); try { const companyId = f.fixture.companyId, issueId = f.issue.id; const leaseId = randomUUID(), providerLeaseId = randomUUID(), workspaceId = randomUUID(), snapshotId = randomUUID(); + if (f.pending) { + await db.update(agentWakeupRequests).set({ requestedAt: new Date(Date.now() - 60_000) }) + .where(eq(agentWakeupRequests.id, f.pending.id)); + await db.update(chatActions).set({ createdAt: new Date(Date.now() - 60_000) }).where(eq(chatActions.id, f.pending.id)); + await db.update(issueComments).set({ createdAt: new Date(Date.now() - 60_000) }) + .where(eq(issueComments.id, String(f.pending.payload.commentId))); + await db.update(heartbeatRuns).set({ finishedAt: new Date(Date.now() - 30_000) }).where(eq(heartbeatRuns.id, f.runId)); + } await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId)); await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, name: "Review workspace", providerType: "cloud_sandbox", mode: "isolated_workspace", strategyType: "project_primary", @@ -65932,6 +65940,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { executionWorkspaceId: workspaceId, phase: "workspace_recovery", status: "succeeded", finishedAt: new Date(), metadata: { recoveryActionId, leaseId, providerLeaseId, snapshotId, generation: 1, sha256: "c".repeat(64), storage: "remote_workspace", epoch: 2 } }); + let sourceRunId: string = f.runId; + if (initiator === "person_after_held_admission") { + const [failed] = await readHeartbeatRunsById(f.runId); + const [held] = await db.insert(heartbeatRuns).values({ companyId, agentId: f.fixture.assignedAgentId, + status: "cancelled", errorCode: "execution_reconciliation_required", runtimeMode: "legacy", issueId, + wakeupRequestId: failed.wakeupRequestId, contextSnapshot: failed.contextSnapshot, finishedAt: new Date(), + createdAt: new Date(failed.createdAt.getTime() + 1_000), + resultJson: { stopReason: "execution_reconciliation_required", timeoutSource: "stale_queued_run_gate", + executionWait: { issueId, recoveryActionId } } }).returning(); + await db.update(agentWakeupRequests).set({ runId: held.id }).where(eq(agentWakeupRequests.id, failed.wakeupRequestId!)); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + sourceRunId = held.id; + } const { getExecutionBlocker } = await import("../services/execution-blocker.js"); const blocker = await getExecutionBlocker(db, companyId, issueId); const interruption = await taskInterruptionPresentation(db, f.issue); @@ -65965,7 +65986,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { let request: Awaited>; try { request = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", - observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry, + observedRunId: sourceRunId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry, ...(initiator.startsWith("person") ? {} : { initiatedBy: "restart_recovery" as const }) }); } finally { await releaseIdentity(); } if (initiator === "restart_recovery_joined_by_person") { @@ -66020,12 +66041,14 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { eq(activityLog.action, "issue.retry_continued_from_saved_work"))); expect(activity).toHaveLength(1); expect(activity[0].details).toMatchObject({ requestId: request.requestId, refusedChatActionId: request.chatActionId }); + if (f.pending) expect((await readAgentWakeupRequestsById(f.pending.id))[0]).toMatchObject({ status: "coalesced", + runId: null, payload: { coalescedIntoWakeupRequestId: request.requestId } }); } finally { await f.service.shutdown(); } }, 90_000); - it.each(["unrecovered_workspace", "access_revoked", "paused_agent"] as const)( + it.each(["unrecovered_workspace", "access_revoked", "paused_agent", "input_after_stop", "input_inside_user_stop"] as const)( "settles a refused GitHub task Retry by its own gates: %s", async gate => { - const f = await currentGitHubTaskRetryFixture(false); + const f = await currentGitHubTaskRetryFixture(gate.startsWith("input_")); try { const companyId = f.fixture.companyId, issueId = f.issue.id; await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", @@ -66038,6 +66061,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: gate === "unrecovered_workspace" ? "recovering" : "recovered", stage: gate === "unrecovered_workspace" ? "restoring_files" : "saving_receipts", attempt: 1, requestIds: [] } } }); + if (gate === "input_inside_user_stop") { + const [failed] = await readHeartbeatRunsById(f.runId); + await db.update(heartbeatRuns).set({ finishedAt: new Date(), resultJson: { cancellation: { source: "operator", + expected: true, initiator: { type: "user", id: "owner-user" }, reason: "Stopped by the user", + recordedAt: failed.finishedAt!.toISOString() } } }).where(eq(heartbeatRuns.id, f.runId)); + } const request = await requestTaskRetry({ db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); const [chat] = await db.update(chatActions).set({ status: "failed", @@ -66054,6 +66083,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { if (gate !== "paused_agent") { expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "failed", runId: null }); expect(await retryRuns()).toHaveLength(0); + if (f.pending) expect((await readAgentWakeupRequestsById(f.pending.id))[0]) + .toMatchObject({ status: "deferred_issue_execution", runId: null }); return; } expect((await readAgentWakeupRequestsById(request.requestId!))[0]).toMatchObject({ status: "deferred_issue_execution", runId: null }); diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e10e666f764..94cd50580e8 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -3,6 +3,7 @@ import { notifyExternalConversationSettlement } from "./external-conversation-se import { writeExecutionControl } from "./execution-control-notifications.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { z } from "zod"; +import { userStopBoundaryAt } from "./run-cancellation.js"; import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; import { randomUUID } from "node:crypto"; import { taskRetryRunAncestry } from "./recovery-run-ancestry.js"; @@ -520,11 +521,12 @@ async function continueRefusedTaskChatRetry(db: Db, wake: typeof agentWakeupRequ const [issue] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, wake.companyId))).for("update"); if (!issue || issue.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(issue.status)) return false; - const [latest] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + const [latest] = await tx.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status, + startedAt: heartbeatRuns.startedAt, errorCode: heartbeatRuns.errorCode, resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, wake.companyId), or(eq(heartbeatRuns.issueId, issueId), eq(heartbeatRuns.nativeIssueId, issueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`))) .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); - if (latest?.id !== intent.sourceRunId) return false; + if (latest?.id !== intent.sourceRunId || !latest.finishedAt) return false; if (intent.initiatedBy) { const [joined] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.agentId, wake.agentId), @@ -535,19 +537,38 @@ async function continueRefusedTaskChatRetry(db: Db, wake: typeof agentWakeupRequ sql`${agentWakeupRequests.payload}#>'{taskRetry,initiatedBy}' is null`)).limit(1); if (!joined) return false; } + const heldBy = latest.status === "cancelled" && !latest.startedAt && latest.errorCode === "execution_reconciliation_required" + ? z.string().uuid().safeParse((latest.resultJson?.executionWait as Record | undefined)?.recoveryActionId) : null; + const [blockingAction] = heldBy?.success ? await tx.select({ evidence: issueRecoveryActions.evidence }) + .from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, wake.companyId), + eq(issueRecoveryActions.sourceIssueId, issueId), eq(issueRecoveryActions.id, heldBy.data))) : []; + const recoveredRunIds = [intent.sourceRunId, ...(typeof blockingAction?.evidence.runId === "string" + ? [blockingAction.evidence.runId] : [])]; const recoveries = await tx.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, wake.companyId), eq(issueRecoveryActions.sourceIssueId, issueId), - sql`${issueRecoveryActions.evidence}->>'runId' = ${intent.sourceRunId}`)); - if (!recoveries.some(({ evidence }) => readSystemRecovery(evidence)?.state === "complete" && - readWorkspaceRecovery(evidence)?.runId === intent.sourceRunId && - readWorkspaceRecovery(evidence)?.status === "recovered")) return false; + inArray(sql`${issueRecoveryActions.evidence}->>'runId'`, recoveredRunIds))); + if (!recoveries.some(({ evidence }) => { + const workspace = readWorkspaceRecovery(evidence); + return readSystemRecovery(evidence)?.state === "complete" && workspace?.status === "recovered" && + workspace.runId === evidence.runId && recoveredRunIds.includes(workspace.runId); + })) return false; if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: wake.companyId, issueId, actorId: intent.actorId, actorSource: intent.actorSource })) return false; const [chatReceipt] = await tx.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId }) .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.id, chat.id))); if (chatReceipt && (chatReceipt.runId || !["failed", "skipped", "cancelled"].includes(chatReceipt.status))) return false; - if (await taskHasUnadmittedChatInput(tx as unknown as Db, { companyId: wake.companyId, issueId, - agentId: wake.agentId, failedRunId: intent.sourceRunId })) return false; + const [unadmitted] = await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, issueId), + eq(issues.companyId, wake.companyId), unadmittedChatWakeupCondition(issues.id, issues.companyId))).limit(1); + if (unadmitted) return false; + const savedInputIds = (await selectTaskRetryInput(tx, { companyId: wake.companyId, issueId, agentId: wake.agentId, + failedRunId: intent.sourceRunId }))?.sourceActionIds ?? []; + const stopBoundary = userStopBoundaryAt(latest); + const savedInputs = savedInputIds.length ? await tx.select({ id: chatActions.id, authorizedAt: issueComments.createdAt }) + .from(chatActions).innerJoin(issueComments, and(eq(issueComments.companyId, chatActions.companyId), + eq(issueComments.issueId, issueId), sql`${issueComments.id}::text = ${chatActions.payload}->>'commentId'`)) + .where(and(eq(chatActions.companyId, wake.companyId), inArray(chatActions.id, savedInputIds))) : []; + if (!stopBoundary || savedInputs.length !== savedInputIds.length || + savedInputs.some(input => input.authorizedAt > stopBoundary)) return false; const { chatActionId: _replaced, initiatedBy: _automatic, ...continuation } = intent; await notifyDeferredExecutionWork(tx, { companyId: wake.companyId, requestId: wake.id }); const [converted] = await tx.update(agentWakeupRequests).set({ updatedAt: new Date(), @@ -558,9 +579,15 @@ async function continueRefusedTaskChatRetry(db: Db, wake: typeof agentWakeupRequ sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' = ${chat.id}`)) .returning({ id: agentWakeupRequests.id }); if (!converted) return false; + if (savedInputIds.length) await tx.update(agentWakeupRequests).set({ status: "coalesced", finishedAt: new Date(), + updatedAt: new Date(), payload: sql`coalesce(${agentWakeupRequests.payload}, '{}'::jsonb) || ${JSON.stringify({ + coalescedIntoWakeupRequestId: wake.id })}::jsonb` }).where(and(eq(agentWakeupRequests.companyId, wake.companyId), + inArray(agentWakeupRequests.id, savedInputIds), eq(agentWakeupRequests.status, "deferred_issue_execution"), + isNull(agentWakeupRequests.runId))); await persistActivity(tx as unknown as Db, { companyId: wake.companyId, actorType: "system", actorId: "task-retry", action: "issue.retry_continued_from_saved_work", entityType: "issue", entityId: issueId, runId: intent.sourceRunId, - details: { requestId: wake.id, sourceRunId: intent.sourceRunId, refusedChatActionId: chat.id } }); + details: { requestId: wake.id, sourceRunId: intent.sourceRunId, refusedChatActionId: chat.id, + ...(savedInputIds.length ? { heldInputRequestIds: savedInputIds } : {}) } }); return true; }); }