diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 5dca17b02fe..ca477a532d7 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -105,6 +105,44 @@ describePostgres("assigned-work invariant sweep", () => { expect(queue.calls).toHaveLength(1); }, 60_000); + it("a wake deferred behind a run that ended long ago is not a pending path; a fresh one still is", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId, { status: "failed", errorCode: "native_restart_recovery_blocked" }); + const deferred = (createdAt: Date) => db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "local-board", payload: { issueId }, createdAt, updatedAt: new Date() }); + await deferred(new Date(Date.now() - 30 * 60_000)); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(issueId); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } }); + + const freshIssue = await f.issue(); + await f.run(freshIssue, { status: "failed", errorCode: "native_restart_recovery_blocked" }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).not.toContain(freshIssue); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } }); + }, 60_000); + + it("a wake claimed long ago that never became a run is not a pending path", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId, { status: "succeeded", errorCode: null, livenessState: null }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + triggerDetail: "system", reason: "issue_children_completed", status: "claimed", payload: { issueId }, + claimedAt: new Date(Date.now() - 15 * 60_000), createdAt: new Date(Date.now() - 40 * 60_000) }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(issueId); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ verdict: { kind: "violation", reason: "no_pending_path" } }); + }, 60_000); + it("an in_progress task whose native run failed needing reconciliation (silently skipped today) gets one continuation", async () => { const f = await seedCompany(); const issueId = await f.issue(); diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 8335e518433..ff58c96927b 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -10,7 +10,7 @@ import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions, } from "../services/execution-recovery-resolution.js"; -import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { PENDING_CONTINUATION_BOUND_MS, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; import { executionControlSchedule } from "../services/execution-control-work.js"; @@ -466,5 +466,26 @@ suite("recovery wake herd", () => { const presentation = await taskInterruptionPresentation(db, issueRow!); expect(presentation).toMatchObject({ canRetry: false, state: "working", responsibility: "system" }); + + const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString(); + await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, held.action.id)); + const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!; + expect(stale.pendingWakes).toBe(facts.pendingWakes - 1); + }, 60_000); + + it("re-drives a continuation held past the hold bound even while another recovery run of its agent stays queued", async () => { + const agentId = await seedCompany(1); + await queuedRecoveryRun(agentId, 1); + const held = await pendingContinuation(agentId, 30); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString(); + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuation,recordedAt}', to_jsonb(${heldSince}::text))` }).where(eq(issueRecoveryActions.id, held.action.id)); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(held.action.id)).toBe("delivered"); }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 8aeb661c102..a8fe99ae905 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -3,10 +3,11 @@ import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agentWakeupRequests, agents, approvals, authUsers, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, - issueComments, issues, nativeRunFinalizations, toolInvocations, + heartbeatRunEvents, issueComments, issues, nativeRunFinalizations, toolInvocations, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; +import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-pacing.js"; import { getExecutionBlocker } from "./execution-blocker.js"; import { FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation, @@ -315,6 +316,35 @@ suite("fresh-continuation fallback", () => { expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered"); }, 60_000); + it("re-drives an isolated continuation pending past the bound itself; a repair wake never replaces it", async () => { + const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); + await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }).where(eq(issues.id, f.issueId)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); + await occupySlot(); + const taskRuns = async () => db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + expect(await sweepWake(f.issueId)).toBeNull(); + expect(await taskRuns()).toHaveLength(0); + const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString(); + await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, f.actionId!)); + expect(await sweepWake(f.issueId)).toBeNull(); + expect(await taskRuns()).toHaveLength(0); + await deliver(); + const runs = await taskRuns(); + expect(runs).toHaveLength(1); + expect(runs[0]!.contextSnapshot).toMatchObject({ freshContinuationIsolation: f.actionId, forceFreshSession: true }); + expect(await readFreshWorkspaceReadiness(db, runs[0]!)).toMatchObject({ mode: "fresh_isolated_workspace", retryIntentId: f.actionId }); + const [record] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + expect(record!.evidence).toMatchObject({ continuationDelivery: "delivered", continuationRunId: runs[0]!.id }); + await sweepWake(f.issueId); + await deliver(); + expect(await taskRuns()).toHaveLength(1); + }, 60_000); + // A gate after the hook must never consume the recovery: it stays pending // and is delivered once the gate clears. it("keeps the continuation pending while the agent's daily run limit declines it, then delivers it", async () => { @@ -803,6 +833,58 @@ suite("fresh-continuation fallback", () => { expect(decision.outcomeEvidence).not.toMatch(adminText); }, 60_000); + it("names the subtasks, comments and documents the cut-off run already created, so the fresh run does not repeat them", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" }); + for (const title of ["SM sub 1 b3d70e77s1", "SM sub 2 b3d70e77s2"]) + await db.insert(issues).values({ companyId, title, status: "todo", assigneeAgentId: agentId, parentId: f.issueId, + originRunId: f.runId } as typeof issues.$inferInsert); + await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorAgentId: agentId, createdByRunId: f.runId, + body: "DONE-0d981334" } as typeof issueComments.$inferInsert); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + const decision = hold!.evidence.executionReconciliation as Record; + expect(decision.outcomeEvidence).toContain('"SM sub 1 b3d70e77s1"'); + expect(decision.outcomeEvidence).toContain('"SM sub 2 b3d70e77s2"'); + expect(decision.outcomeEvidence).toContain('"DONE-0d981334"'); + expect(decision.outcomeEvidence).toContain("Do not create or post them again"); + expect(hold!.evidence.freshContinuation).toMatchObject({ createdObjects: { subtasks: expect.arrayContaining([expect.stringContaining("SM sub 1")]) } }); + }, 60_000); + + it("names the follow-up the cut-off run had read and its own final comment, so the agent answers once (t003)", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" }); + const [followUp] = await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorType: "user", + authorUserId: "board-user", body: "Additional requirement: include the line FOLLOWUP-0d981334" } as typeof issueComments.$inferInsert).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, runId: f.runId, agentId, seq: 1, eventType: "task_history.read", + stream: "system", level: "info", payload: { comments: { [followUp!.id]: followUp!.updatedAt.toISOString() } }, + } as typeof heartbeatRunEvents.$inferInsert); + await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorAgentId: agentId, createdByRunId: f.runId, + body: "DONE-0d981334" } as typeof issueComments.$inferInsert); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + const note = (hold!.evidence.executionReconciliation as Record).outcomeEvidence; + expect(note).toContain("had already read these messages"); + expect(note).toContain("FOLLOWUP-0d981334"); + expect(note).toContain('posted comments "DONE-0d981334"'); + expect(note.indexOf("had already read")).toBeLessThan(note.indexOf("DONE-0d981334")); + }, 60_000); + + it("does not list a follow-up edited after the cut-off run read it as already read; it is marked as changed", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" }); + const [followUp] = await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorType: "user", + authorUserId: "board-user", body: "Additional requirement: include the line FOLLOWUP-OLD" } as typeof issueComments.$inferInsert).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, runId: f.runId, agentId, seq: 1, eventType: "task_history.read", + stream: "system", level: "info", payload: { comments: { [followUp!.id]: followUp!.updatedAt.toISOString() } }, + } as typeof heartbeatRunEvents.$inferInsert); + await db.update(issueComments).set({ body: "Additional requirement: include the line FOLLOWUP-NEW", + updatedAt: new Date(followUp!.updatedAt.getTime() + 60_000) }).where(eq(issueComments.id, followUp!.id)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + const note = (hold!.evidence.executionReconciliation as Record).outcomeEvidence; + expect(note).not.toContain("had already read"); + expect(note).toContain("edited these messages after that run read them"); + expect(note).toContain("FOLLOWUP-NEW"); + }, 60_000); + it.each([true, false])("relies on saved input only while it still holds an undelivered message (already taken by the cut-off run: %s)", async taken => { // R2 cell t003: the cut-off run had taken the follow-up into its turn; the fallback chose saved_input, // promotion settled the message as delivered, and no run followed. diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index adc6eb81df5..c28825ea6b0 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -1,13 +1,14 @@ import { and, asc, desc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { - agentWakeupRequests, agents, approvals, companies, executionWorkspaces, heartbeatRuns, issueApprovals, issueRecoveryActions, - issueThreadInteractions, issues, nativeRunFinalizations, toolInvocations, type Db, + agentWakeupRequests, agents, approvals, companies, documentRevisions, documents, executionWorkspaces, heartbeatRuns, issueApprovals, + issueComments, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, toolInvocations, type Db, } from "@paperclipai/db"; import { EXECUTION_RECONCILIATION_CAUSES, readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionReconciliation, } from "@paperclipai/shared"; import { logger } from "../middleware/logger.js"; import { logActivity } from "./activity-log.js"; +import { taskHistoryReadsForRun } from "./queued-comment-delivery.js"; import { undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js"; import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; import { FRESH_CONTINUATION_SUPERSEDED_KEY } from "../agent-run-cancellation.js"; @@ -197,8 +198,48 @@ async function unverifiedActions(tx: Db, run: typeof heartbeatRuns.$inferSelect) return [...names].slice(0, 20); } +export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[]; + editedFollowUps?: string[] }; + +async function taskHistoryReadsOfRun(tx: Db, run: Pick) { + const reads = await taskHistoryReadsForRun(tx, { companyId: run.companyId, runId: run.id }); + return new Map(reads.map(read => [read.commentId, read])); +} + +const quoted = (value: string, max = 80) => `"${value.replace(/\s+/g, " ").trim().slice(0, max)}${value.length > max ? "..." : ""}"`; + +async function createdObjects(tx: Db, run: typeof heartbeatRuns.$inferSelect): Promise { + const [children, comments, revisions] = await Promise.all([ + tx.select({ identifier: issues.identifier, title: issues.title }).from(issues).where(and( + eq(issues.companyId, run.companyId), eq(issues.originRunId, run.id), isNull(issues.hiddenAt))) + .orderBy(asc(issues.createdAt)).limit(20), + tx.select({ body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.createdByRunId, run.id), isNull(issueComments.deletedAt))) + .orderBy(asc(issueComments.createdAt)).limit(10), + tx.selectDistinct({ title: documents.title }).from(documentRevisions) + .innerJoin(documents, eq(documents.id, documentRevisions.documentId)) + .where(and(eq(documentRevisions.companyId, run.companyId), eq(documentRevisions.createdByRunId, run.id))).limit(10), + ]); + const reads = await taskHistoryReadsOfRun(tx, run); + const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body, updatedAt: issueComments.updatedAt }) + .from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]), + eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : []; + const unchanged = (comment: (typeof readComments)[number]) => comment.updatedAt.getTime() <= reads.get(comment.id)!.revision.getTime(); + const edited = readComments.filter(comment => !unchanged(comment)); + return { + readFollowUps: readComments.filter(unchanged) + .map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.readAt.toISOString().slice(11, 16)} UTC)`), + ...(edited.length ? { editedFollowUps: edited.map(comment => quoted(comment.body)) } : {}), + subtasks: children.map(child => `${child.identifier ?? "subtask"} ${quoted(child.title)}`), + comments: comments.map(comment => quoted(comment.body)), + documents: revisions.map(revision => quoted(revision.title ?? "untitled document")), + }; +} + /** The opening context of the continuation. It names no person who must act. */ -export function freshContinuationNote(input: { runId: string; unverified: string[]; workspace: "restored_last_save" | "kept" }) { +export function freshContinuationNote(input: { runId: string; unverified: string[]; workspace: "restored_last_save" | "kept"; + created?: FreshContinuationCreatedObjects }) { const actions = input.unverified.length ? `These actions of that run have unverified outcomes: ${input.unverified.join(", ")}. For each one, first check whether it took effect.` : "No action of that run is recorded with an unknown outcome; check the task's recent history before repeating any side effect."; @@ -211,7 +252,21 @@ export function freshContinuationNote(input: { runId: string; unverified: string + "If an action did not take effect, or repeating it is safe, do it again and continue the task. " + "If repeating it would be a governed or irreversible action, request it through the normal approval step. " + "Nothing about the previous attempt needs to be restored before you continue."; - return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files} ${actions} ${guidance}`; + const created = input.created; + const read = created?.readFollowUps?.length + ? ` That run had already read these messages from a person: ${created.readFollowUps.join(", ")}. Answer each once; what it did after reading is below.` + : ""; + const edited = created?.editedFollowUps?.length + ? ` A person edited these messages after that run read them, so it never saw their current text; treat them as new: ${created.editedFollowUps.join(", ")}.` + : ""; + const existing = created &&(created.subtasks.length || created.comments.length || created.documents.length) + ? ` That run already did these, and they exist now: ${[ + created.subtasks.length ? `created subtasks ${created.subtasks.join(", ")}` : null, + created.comments.length ? `posted comments ${created.comments.join(", ")}` : null, + created.documents.length ? `saved documents ${created.documents.join(", ")}` : null, + ].filter(Boolean).join("; ")}. Do not create or post them again; read them and continue from where they leave off.` + : ""; + return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${edited}${existing} ${actions} ${guidance}`; } const USER_TEXT = "The previous run was cut off. Paperclip continues the task in a fresh run from its last saved work."; @@ -546,7 +601,8 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // continuation runs first, binds the new workspace, and saved input follows. if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded"; const unverified = await unverifiedActions(tx, run); - const note = freshContinuationNote({ runId: run.id, unverified, + const created = await createdObjects(tx, run); + const note = freshContinuationNote({ runId: run.id, unverified, created, workspace: isolation === "writer_ended" ? "kept" : "restored_last_save" }); const decision: ExecutionReconciliation & { providerStopProof: "writer_ended" | "isolated" } = { runId: run.id, providerStopped: true, // Only a proven end is a stop. `isolated` means exactly: the old run's agent @@ -563,7 +619,7 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< const freshContinuation = { schema: FRESH_CONTINUATION_SCHEMA, sourceRunId: run.id, trigger: input.trigger, failureCode: run.errorCode ?? null, delivery, recordedAt: now.toISOString(), isolation, stopProof: isolation === "writer_ended" ? "writer_ended" : fenced ? "unproven_epoch_abandoned" : "unproven_isolated", - unverifiedActions: unverified, ...(fenced ? { workspace: fenced } : {}) }; + unverifiedActions: unverified, createdObjects: created, ...(fenced ? { workspace: fenced } : {}) }; const reconciliation = { ...decision, actorId: FRESH_CONTINUATION_ACTOR, recordedAt: now.toISOString() }; // Who, what, why and when. A fresh run is not a replay: the interrupted turn // is not re-sent, no tool call or write is re-executed, and any governed @@ -872,14 +928,15 @@ export async function settleCoalescedFreshContinuationHolds(db: Db, input: { com sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'coalesced'`)); } -/** A fresh continuation of this task is recorded and its run does not exist yet. */ export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string) { const [row] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`)).limit(1); - return Boolean(row); + if (!row) return false; + await notifyDeliveryWork(db, DELIVERY_QUEUES.executionContinuation); + return true; } /** Newest-first list of fresh continuations of one task (diagnostics and tests). */ diff --git a/server/src/services/recovery-continuation-pacing.ts b/server/src/services/recovery-continuation-pacing.ts index 266e795b15f..d0d9b9e18c2 100644 --- a/server/src/services/recovery-continuation-pacing.ts +++ b/server/src/services/recovery-continuation-pacing.ts @@ -1,6 +1,7 @@ export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; export const RECOVERY_CONTINUATION_DELIVERY_BOUND_MS = 15 * 60_000; +export const PENDING_CONTINUATION_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS + 2 * 60_000; export function recoveryContinuationHeldSince(input: { evidence: Record; diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts index 6be75f2a9d0..00b9aaff2a0 100644 --- a/server/src/services/recovery/assigned-work-invariant.test.ts +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -8,6 +8,7 @@ import { SETTLING_GRACE_MS, SETTLING_MAX_MS, STALE_BLOCK_MS, + SYSTEM_RECOVERY_WORKING_BOUND_MS, VIOLATION_REASONS, type AssignedWorkFacts, } from "./assigned-work-invariant.js"; @@ -74,7 +75,8 @@ describe("assigned-work invariant", () => { ["pending_handoff", { pendingHandoff: true }], ["session_goal", { sessionGoal: "active" }], ["scheduled_monitor", { issue: { monitorNextCheckAt: new Date(now.getTime() + 60_000) } }], - ["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false } }], + ["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false }, + latestRun: { id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", finishedAt: ago(5 * 60_000), settlingUntil: null, operatorStopped: false } }], ]; for (const [intent, overrides] of cases) { it(`accepts ${intent}`, () => { @@ -82,6 +84,16 @@ describe("assigned-work invariant", () => { }); } + it("stops believing system recovery is working once the last run ended longer ago than the bound", () => { + const working = { responsibility: "system" as const, state: "working", userAction: null, nextRetryAt: null, canRetry: false }; + const run = (msAgo: number) => ({ id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", + finishedAt: ago(msAgo), settlingUntil: null, operatorStopped: false }); + expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS - 60_000) }), now)) + .toEqual({ kind: "ok", clause: "pending_intent", intent: "system_recovery_working" }); + expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS + 60_000) }), now)) + .toMatchObject({ kind: "violation", reason: "no_pending_path" }); + }); + it("does not count a monitor or retry whose time has passed", () => { expect(evaluateAssignedWork(facts({ issue: { monitorNextCheckAt: ago(1) } }), now)).toMatchObject({ kind: "violation" }); expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "retry_scheduled", userAction: null, diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index f9d030ea85f..9bc465ea4e4 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -100,6 +100,7 @@ const PROVIDER_LIMIT_CAUSES = new Set(["provider_quota", "native_provider_usage_ export const SETTLING_GRACE_MS = 90_000; /** Upper bound on how long another server's lease can keep a finished run settling. */ export const SETTLING_MAX_MS = 5 * 60_000; +export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 17 * 60_000; /** Blocked tasks older than this are reported, never continued automatically. */ export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000; /** Automatic continuations per task, shared with the server-restart lane. */ @@ -231,7 +232,8 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor"; const interruption = facts.interruption; if (interruption?.responsibility === "system") { - if (interruption.state === "working") return "system_recovery_working"; + if (interruption.state === "working" && !(facts.latestRun?.finishedAt + && now.getTime() - facts.latestRun.finishedAt.getTime() > SYSTEM_RECOVERY_WORKING_BOUND_MS)) return "system_recovery_working"; if (interruption.state === "retry_scheduled" && interruption.nextRetryAt && Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry"; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index e0225712071..2c5452b71c1 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -29,6 +29,7 @@ import { issueTreeControlService } from "../issue-tree-control.js"; import { taskInterruptionPresentations } from "../task-interruption-presentation.js"; import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; +import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; import { continuationActivityMessage, continuationAllowance, @@ -57,6 +58,8 @@ export const ASSIGNED_WORK_REPORT_LIMIT = 2_000; const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const; const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"]; +const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000; +export { PENDING_CONTINUATION_BOUND_MS }; const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]; const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"]; @@ -195,6 +198,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { database.select({ issueId: sql`${agentWakeupRequests.payload}->>'issueId'`, count: sql`count(*)::int` }) .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES), + sql`(${agentWakeupRequests.status} = 'queued' + or (${agentWakeupRequests.status} = 'deferred_issue_execution' + and ${agentWakeupRequests.createdAt} > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval) + -- A claimed wake becomes a run within seconds; one claimed long ago with no run never will. + or (${agentWakeupRequests.status} = 'claimed' and (${agentWakeupRequests.runId} is not null + or coalesce(${agentWakeupRequests.claimedAt}, ${agentWakeupRequests.createdAt}) + > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval)))`, sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`)) .groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`), database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids), @@ -281,7 +291,10 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { agentId: issueRecoveryActions.returnOwnerAgentId }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, + ${issueRecoveryActions.updatedAt}::text)::timestamptz + > ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval`)); const wakeCounts = counts(wakes); for (const continuation of pendingContinuations) { const issue = issueRows.find(row => row.id === continuation.issueId); @@ -424,7 +437,10 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`, sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} and (w.payload->>'issueId') = ${issues.id}::text - and w.status in ('queued', 'deferred_issue_execution', 'claimed'))`, + and (w.status = 'queued' + or (w.status = 'deferred_issue_execution' and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz) + or (w.status = 'claimed' and (w.run_id is not null + or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz))))`, sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} and t.issue_id = ${issues.id} and t.status = 'pending' and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`,