diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 21a67499d96..8cf3d65df14 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -166,6 +166,35 @@ suite("fresh-continuation fallback", () => { // Staging: the person had already pressed Retry, and the automatic successor // could not carry that authorization forward. That is the system's failure, // not the person's: the task continues under its responsible user. + it("starts the continuation when the failed run's exhaustion hold lands after the continuation was recorded", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_runner_process_exited", + error: "native_session_recovery_failed: Error: runner_local_connect_failed: runnerd exited unexpectedly" }); + await settleUnrecoverableExecutions(db); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "active", + ownerType: "board", returnOwnerAgentId: agentId, cause: "native_session_retry_exhausted", fingerprint: `native-retry:${f.runId}`, + nextAction: "Automatic recovery stopped.", maxAttempts: 3, + evidence: { runId: f.runId, coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" } }); + await deliver(); + await continueHeldExecutions(db); await settleUnrecoverableExecutions(db); await deliver(); + expect(await successors(f.runId)).toHaveLength(1); + }, 60_000); + + it("keeps a person's newer block when the failed run's exhaustion hold lands after the continuation was recorded", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_runner_process_exited", + error: "native_session_recovery_failed: Error: runner_local_connect_failed: runnerd exited unexpectedly" }); + await settleUnrecoverableExecutions(db); + const decisionId = randomUUID(); + await db.update(issues).set({ status: "blocked", statusVersion: 9, lastStatusDecisionId: decisionId, blockedTransitionAt: new Date() }) + .where(eq(issues.id, f.issueId)); + const [late] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "active", + ownerType: "board", returnOwnerAgentId: agentId, cause: "native_session_retry_exhausted", fingerprint: `native-retry:${f.runId}`, + nextAction: "Automatic recovery stopped.", maxAttempts: 3, + evidence: { runId: f.runId, coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted", nativeFailureBlock: { runId: f.runId, statusVersion: 4 } } }).returning(); + await settleUnrecoverableExecutions(db); + expect(await issue(f.issueId)).toMatchObject({ status: "blocked", lastStatusDecisionId: decisionId }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, late!.id)))[0]).toMatchObject({ status: "resolved", ownerType: "system" }); + }, 60_000); + it("continues a task whose automatic retry lost the person's continuation authorization", async () => { expect(PERSON_OWNED_FAILURE_CODES).not.toContain("continuation_user_authorization_missing"); const f = await unfinishedRun({ runtime: "legacy", errorCode: "continuation_user_authorization_missing", diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 092084af912..27b9b3709c9 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -339,7 +339,10 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< or(sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, sql`${issueRecoveryActions.evidence}->>'sourceRunId' = ${run.id}`))) .orderBy(asc(issueRecoveryActions.createdAt), asc(issueRecoveryActions.id)).for("update"); const prior = holds.find(hold => object(hold.evidence.freshContinuation).sourceRunId === run.id); - if (prior) return { kind: "already_continued", recoveryActionId: prior.id }; + if (prior) { + await settleLateHolds(tx, { companyId, issue, run, prior, holds: holds.filter(isOwnedReconciliationHold), now }); + return { kind: "already_continued", recoveryActionId: prior.id }; + } // The holds this decision owns: reconciliation holds that still block the // task. Workspace restore/repair holds stay with workspace recovery, and // resolved history is never rewritten. @@ -593,6 +596,36 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< return { kind: "continued", recoveryActionId: primaryId, delivery, isolation }; } +async function settleLateHolds(tx: Db, input: { companyId: string; issue: typeof issues.$inferSelect; run: typeof heartbeatRuns.$inferSelect; + prior: typeof issueRecoveryActions.$inferSelect; holds: Array; now: Date }) { + const { companyId, issue, run, prior, holds, now } = input; + if (!holds.length) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); + await notifyStatusCardIssueChange(tx, companyId); + await notifyRoutineExecutionStatus(tx, issue.id, companyId); + const ownsBlock = issue.status === "blocked" && holds.some(hold => holdOwnsBlockedStatus(hold, issue, run)); + const delivery = String(prior.evidence.continuationDelivery); + const followed = ["delivered", "invalidated"].includes(delivery) ? delivery : "coalesced"; + for (const hold of holds) { + const { automaticRecovery, [FRESH_CONTINUATION_CHECK_KEY]: _check, ...evidence } = hold.evidence; + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, + ownerType: "system", ownerAgentId: null, ownerUserId: null, wakePolicy: null, monitorPolicy: null, nextAction: USER_TEXT, + resolutionNote: "continued_fresh_from_last_saved", + evidence: { ...evidence, ...(automaticRecovery !== undefined ? { automaticRecoveryHistory: automaticRecovery } : {}), + continuationDelivery: followed, coalescedIntoRecoveryActionId: prior.id, + ...(typeof prior.evidence.continuationRunId === "string" ? { continuationRunId: prior.evidence.continuationRunId } : {}) }, + }).where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.id, hold.id))); + } + if (ownsBlock) { + await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, + blockedOwnerNotifiedAt: null, updatedAt: now }).where(and(eq(issues.companyId, companyId), eq(issues.id, issue.id))); + await logActivity(tx, { companyId, actorType: "system", actorId: FRESH_CONTINUATION_ACTOR, action: "issue.updated", + entityType: "issue", entityId: issue.id, agentId: run.agentId, runId: run.id, + details: { status: "todo", previousStatus: "blocked", reason: "fresh_continuation", recoveryActionId: prior.id, + _previous: { status: "blocked" } } }); + } +} + /** Records why a hold was not continued yet, and when to look again. */ export async function recordFreshContinuationCheck(db: Db, input: { companyId: string; actionId: string }, result: FreshContinuationResult, now = new Date()) {