diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index d40e46f45ef..62ae2529b0d 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -299,4 +299,19 @@ suite("recovery wake herd", () => { const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!; expect(stale.pendingWakes).toBe(facts.pendingWakes - 1); }, 60_000); + + it("re-drives a continuation held past the hold bound even while another recovery run of its agent stays queued", async () => { + const agentId = await seedCompany(1); + await queuedRecoveryRun(agentId, 1); + const held = await pendingContinuation(agentId, 30); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString(); + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuation,recordedAt}', to_jsonb(${heldSince}::text))` }).where(eq(issueRecoveryActions.id, held.action.id)); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(held.action.id)).toBe("delivered"); + }, 60_000); }); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index a14870dcbe5..5cddd11cce9 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -42,9 +42,10 @@ import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCh import { alias } from "drizzle-orm/pg-core"; import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; +import { RECOVERY_CONTINUATION_MAX_HOLD_MS } from "./recovery-continuation-bounds.js"; export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; -export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; +export { RECOVERY_CONTINUATION_MAX_HOLD_MS }; const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8; /** An operator records observed outcomes; this is not permission to blindly retry. */ @@ -389,11 +390,15 @@ export async function deliverReconciledExecutions( seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined, companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, - sql`not exists (select 1 from ${heartbeatRuns} queued_recovery + // One queued recovery run per agent paces the rest, but only within + // the hold bound: a continuation held past it is admitted like one + // held behind ordinary work (agentHasRoom), or it would wait as long + // as that other run stays queued. + sql`(${heldSince} <= ${heldBound}::timestamptz or not exists (select 1 from ${heartbeatRuns} queued_recovery where queued_recovery.company_id = ${issueRecoveryActions.companyId} and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} and queued_recovery.status = 'queued' - and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)}))`, + and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`, ), ) .orderBy( diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 35f6568194e..67d9bbb51f3 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -7,6 +7,7 @@ import { } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; +import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js"; import { getExecutionBlocker } from "./execution-blocker.js"; import { FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation, @@ -315,6 +316,30 @@ suite("fresh-continuation fallback", () => { expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered"); }, 60_000); + it("lets a repair wake become the one successor once a pending continuation is past the bound, and coalesces it within the bound", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_continuation_requires_reconciliation", + error: "runner_remote_process_identity_unavailable", hold: "settled_blocked" }); + await occupySlot(); + expect(await sweepWake(f.issueId)).toBeNull(); + const record = (await pendingRecord(f.issueId))!; + expect(record.evidence.continuationDelivery).toBe("pending"); + const taskRuns = async () => db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + // Within the bound, the repair wake is coalesced into the pending record. + expect(await sweepWake(f.issueId)).toBeNull(); + expect(await taskRuns()).toHaveLength(0); + // Delivery is stuck past the bound: the next repair wake becomes the continuation. + const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString(); + await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, record.id)); + expect(await sweepWake(f.issueId)).not.toBeNull(); + expect(await taskRuns()).toHaveLength(1); + // Delivery then settles the record as delivered by that run; still exactly one run. + await deliver(); + expect(await taskRuns()).toHaveLength(1); + expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered"); + }, 60_000); + // A gate after the hook must never consume the recovery: it stays pending // and is delivered once the gate clears. it("keeps the continuation pending while the agent's daily run limit declines it, then delivers it", async () => { @@ -826,6 +851,24 @@ suite("fresh-continuation fallback", () => { expect(note.indexOf("had already read")).toBeLessThan(note.indexOf("DONE-0d981334")); }, 60_000); + it("does not list a follow-up edited after the cut-off run read it as already read; it is marked as changed", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" }); + const [followUp] = await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorType: "user", + authorUserId: "board-user", body: "Additional requirement: include the line FOLLOWUP-OLD" } as typeof issueComments.$inferInsert).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, runId: f.runId, agentId, seq: 1, eventType: "task_history.read", + stream: "system", level: "info", payload: { comments: { [followUp!.id]: followUp!.updatedAt.toISOString() } }, + } as typeof heartbeatRunEvents.$inferInsert); + // The person edits the follow-up after the run read it. + await db.update(issueComments).set({ body: "Additional requirement: include the line FOLLOWUP-NEW", + updatedAt: new Date(followUp!.updatedAt.getTime() + 60_000) }).where(eq(issueComments.id, followUp!.id)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + const note = (hold!.evidence.executionReconciliation as Record).outcomeEvidence; + expect(note).not.toContain("had already read"); + expect(note).toContain("edited these messages after that run read them"); + expect(note).toContain("FOLLOWUP-NEW"); + }, 60_000); + it.each([ { title: "a pending approval", setup: "approval" as const, errorCode: "native_continuation_requires_reconciliation" }, { title: "a credential the person has not shared", setup: "none" as const, errorCode: "configuration_incomplete" }, diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 8542e508ff0..ed530cac8bc 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -24,6 +24,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; +import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js"; /** * Fresh-continuation fallback: exact resumption of an unfinished run is an @@ -196,13 +197,15 @@ async function unverifiedActions(tx: Db, run: typeof heartbeatRuns.$inferSelect) return [...names].slice(0, 20); } -export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[] }; +export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[]; + /** Messages the run read that a person edited afterwards: the run never saw the current text. */ + editedFollowUps?: string[] }; -/** Comment ids the run read through get_task_history, with the read time (the - * queued-comment owner's `task_history.read` reader). */ +/** Comment ids the run read through get_task_history, with the read time and + * the comment revision it read (the queued-comment owner's `task_history.read` reader). */ async function taskHistoryReadsOfRun(tx: Db, run: Pick) { const reads = await taskHistoryReadsForRun(tx, { companyId: run.companyId, runId: run.id }); - return new Map(reads.map(read => [read.commentId, read.readAt])); + return new Map(reads.map(read => [read.commentId, read])); } const quoted = (value: string, max = 80) => `"${value.replace(/\s+/g, " ").trim().slice(0, max)}${value.length > max ? "..." : ""}"`; @@ -222,11 +225,18 @@ async function createdObjects(tx: Db, run: typeof heartbeatRuns.$inferSelect): P .where(and(eq(documentRevisions.companyId, run.companyId), eq(documentRevisions.createdByRunId, run.id))).limit(10), ]); const reads = await taskHistoryReadsOfRun(tx, run); - const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]), - eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : []; + const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body, updatedAt: issueComments.updatedAt }) + .from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]), + eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : []; + // Only the revision the run read counts as read: a message edited since then + // is quoted as changed, so the agent treats its current text as new. + const unchanged = (comment: (typeof readComments)[number]) => comment.updatedAt.getTime() <= reads.get(comment.id)!.revision.getTime(); + const edited = readComments.filter(comment => !unchanged(comment)); return { - readFollowUps: readComments.map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.toISOString().slice(11, 16)} UTC)`), + readFollowUps: readComments.filter(unchanged) + .map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.readAt.toISOString().slice(11, 16)} UTC)`), + ...(edited.length ? { editedFollowUps: edited.map(comment => quoted(comment.body)) } : {}), subtasks: children.map(child => `${child.identifier ?? "subtask"} ${quoted(child.title)}`), comments: comments.map(comment => quoted(comment.body)), documents: revisions.map(revision => quoted(revision.title ?? "untitled document")), @@ -252,14 +262,17 @@ export function freshContinuationNote(input: { runId: string; unverified: string const read = created?.readFollowUps?.length ? ` That run had already read these messages from a person: ${created.readFollowUps.join(", ")}. Answer each once; what it did after reading is below.` : ""; - const existing = created && (created.subtasks.length || created.comments.length || created.documents.length) + const edited = created?.editedFollowUps?.length + ? ` A person edited these messages after that run read them, so it never saw their current text; treat them as new: ${created.editedFollowUps.join(", ")}.` + : ""; + const existing = created &&(created.subtasks.length || created.comments.length || created.documents.length) ? ` That run already did these, and they exist now: ${[ created.subtasks.length ? `created subtasks ${created.subtasks.join(", ")}` : null, created.comments.length ? `posted comments ${created.comments.join(", ")}` : null, created.documents.length ? `saved documents ${created.documents.join(", ")}` : null, ].filter(Boolean).join("; ")}. Do not create or post them again; read them and continue from where they leave off.` : ""; - return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${existing} ${actions} ${guidance}`; + return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${edited}${existing} ${actions} ${guidance}`; } const USER_TEXT = "The previous run was cut off. Paperclip continues the task in a fresh run from its last saved work."; @@ -874,12 +887,19 @@ export async function settleCoalescedFreshContinuationHolds(db: Db, input: { com sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'coalesced'`)); } -/** A fresh continuation of this task is recorded and its run does not exist yet. */ -export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string) { +/** A fresh continuation of this task is recorded, its run does not exist yet, + * and delivery still owns it. Past PENDING_CONTINUATION_BOUND_MS delivery has + * had its chance (it re-drives a record held past its hold bound first), so the + * record no longer absorbs a repair wake: that wake becomes the task's one + * successor, and delivery then settles the record as delivered by it. */ +export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string, now = new Date()) { const [row] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, + ${issueRecoveryActions.updatedAt}::text)::timestamptz + > ${new Date(now.getTime() - PENDING_CONTINUATION_BOUND_MS).toISOString()}::timestamptz`, sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`)).limit(1); return Boolean(row); } diff --git a/server/src/services/recovery-continuation-bounds.ts b/server/src/services/recovery-continuation-bounds.ts new file mode 100644 index 00000000000..44a10373751 --- /dev/null +++ b/server/src/services/recovery-continuation-bounds.ts @@ -0,0 +1,17 @@ +/** + * How long a recorded recovery continuation may wait for delivery. One module + * owns both numbers so delivery, wake admission and the assigned-work + * invariant agree on when a pending continuation stops being the task's path. + */ + +/** Delivery pacing holds a continuation back at most this long: past it the + * delivery worker admits it even when the agent's queue is full or another + * recovery run of the agent is queued. */ +export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; + +/** A continuation still pending past this bound (the hold bound plus 2 minutes + * of 15-second delivery passes) is not going to be delivered: the invariant no + * longer counts it as pending, and wake admission no longer coalesces a repair + * wake into it, so that wake becomes the task's one successor. Delivery + * re-drives first, so within the bound there is exactly one owner. */ +export const PENDING_CONTINUATION_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS + 2 * 60_000; diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 498d08dfdaf..cad6d18a9a3 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -29,6 +29,7 @@ import { issueTreeControlService } from "../issue-tree-control.js"; import { taskInterruptionPresentations } from "../task-interruption-presentation.js"; import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; +import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-bounds.js"; import { continuationActivityMessage, continuationAllowance, @@ -59,10 +60,9 @@ const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const; const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"]; /** How long a deferred wake (no run holding its task) or a claimed wake with no run counts as pending. */ const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000; -/** How long a held continuation (paced delivery) counts as pending: the pacing's - * admission bound (RECOVERY_CONTINUATION_MAX_HOLD_MS, 5 min) plus 2 minutes of - * 15-second delivery passes. */ -export const PENDING_CONTINUATION_BOUND_MS = 7 * 60_000; +/** How long a held continuation (paced delivery) counts as pending; shared with + * wake admission and delivery. */ +export { PENDING_CONTINUATION_BOUND_MS }; const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]; const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];