fix(recovery): repair a continuation whose delivery is stuck; an edited follow-up is not shown as read

- One module (recovery-continuation-bounds.ts) owns the hold bound (5 min)
  and the pending bound (hold bound + 2 min); the invariant, wake admission
  and delivery read both from it.
- Delivery: a continuation held past the hold bound is no longer skipped
  because another recovery run of its agent is queued.
- Admission: hasPendingFreshContinuation counts a record only within the
  pending bound. Past it a repair wake becomes the task's run; delivery's
  newer-run check settles the record as delivered by it (one successor).
- Opening note: only the comment revision the cut-off run read counts as
  read; a follow-up edited afterwards is listed as changed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 20:50:08 -06:00
1 parent cfd42b0516
commit b76111e45f
6 files changed
+119 -19

No files matched your search

@@ -299,4 +299,19 @@ suite("recovery wake herd", () => {
const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
expect(stale.pendingWakes).toBe(facts.pendingWakes - 1);
}, 60_000);
it("re-drives a continuation held past the hold bound even while another recovery run of its agent stays queued", async () => {
const agentId = await seedCompany(1);
await queuedRecoveryRun(agentId, 1);
const held = await pendingContinuation(agentId, 30);
const wake = fakeWake();
await deliverReconciledExecutions(db, wake as never);
expect(wake).not.toHaveBeenCalled();
const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString();
await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
'{freshContinuation,recordedAt}', to_jsonb(${heldSince}::text))` }).where(eq(issueRecoveryActions.id, held.action.id));
await deliverReconciledExecutions(db, wake as never);
expect(wake).toHaveBeenCalledTimes(1);
expect(await deliveryOf(held.action.id)).toBe("delivered");
}, 60_000);
});
@@ -42,9 +42,10 @@ import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCh
import { alias } from "drizzle-orm/pg-core";
import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js";
import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js";
import { RECOVERY_CONTINUATION_MAX_HOLD_MS } from "./recovery-continuation-bounds.js";
export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5;
export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000;
export { RECOVERY_CONTINUATION_MAX_HOLD_MS };
const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8;
/** An operator records observed outcomes; this is not permission to blindly retry. */
@@ -389,11 +390,15 @@ export async function deliverReconciledExecutions(
seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined,
seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined,
companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined,
sql`not exists (select 1 from ${heartbeatRuns} queued_recovery
// One queued recovery run per agent paces the rest, but only within
// the hold bound: a continuation held past it is admitted like one
// held behind ordinary work (agentHasRoom), or it would wait as long
// as that other run stays queued.
sql`(${heldSince} <= ${heldBound}::timestamptz or not exists (select 1 from ${heartbeatRuns} queued_recovery
where queued_recovery.company_id = ${issueRecoveryActions.companyId}
and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId}
and queued_recovery.status = 'queued'
and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)}))`,
and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`,
),
)
.orderBy(
@@ -7,6 +7,7 @@ import {
} from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js";
import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js";
import { getExecutionBlocker } from "./execution-blocker.js";
import {
FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation,
@@ -315,6 +316,30 @@ suite("fresh-continuation fallback", () => {
expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered");
}, 60_000);
it("lets a repair wake become the one successor once a pending continuation is past the bound, and coalesces it within the bound", async () => {
const f = await unfinishedRun({ runtime: "native", errorCode: "native_continuation_requires_reconciliation",
error: "runner_remote_process_identity_unavailable", hold: "settled_blocked" });
await occupySlot();
expect(await sweepWake(f.issueId)).toBeNull();
const record = (await pendingRecord(f.issueId))!;
expect(record.evidence.continuationDelivery).toBe("pending");
const taskRuns = async () => db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`));
// Within the bound, the repair wake is coalesced into the pending record.
expect(await sweepWake(f.issueId)).toBeNull();
expect(await taskRuns()).toHaveLength(0);
// Delivery is stuck past the bound: the next repair wake becomes the continuation.
const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString();
await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
'{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, record.id));
expect(await sweepWake(f.issueId)).not.toBeNull();
expect(await taskRuns()).toHaveLength(1);
// Delivery then settles the record as delivered by that run; still exactly one run.
await deliver();
expect(await taskRuns()).toHaveLength(1);
expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered");
}, 60_000);
// A gate after the hook must never consume the recovery: it stays pending
// and is delivered once the gate clears.
it("keeps the continuation pending while the agent's daily run limit declines it, then delivers it", async () => {
@@ -826,6 +851,24 @@ suite("fresh-continuation fallback", () => {
expect(note.indexOf("had already read")).toBeLessThan(note.indexOf("DONE-0d981334"));
}, 60_000);
it("does not list a follow-up edited after the cut-off run read it as already read; it is marked as changed", async () => {
const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" });
const [followUp] = await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorType: "user",
authorUserId: "board-user", body: "Additional requirement: include the line FOLLOWUP-OLD" } as typeof issueComments.$inferInsert).returning();
await db.insert(heartbeatRunEvents).values({ companyId, runId: f.runId, agentId, seq: 1, eventType: "task_history.read",
stream: "system", level: "info", payload: { comments: { [followUp!.id]: followUp!.updatedAt.toISOString() } },
} as typeof heartbeatRunEvents.$inferInsert);
// The person edits the follow-up after the run read it.
await db.update(issueComments).set({ body: "Additional requirement: include the line FOLLOWUP-NEW",
updatedAt: new Date(followUp!.updatedAt.getTime() + 60_000) }).where(eq(issueComments.id, followUp!.id));
expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" });
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!));
const note = (hold!.evidence.executionReconciliation as Record<string, string>).outcomeEvidence;
expect(note).not.toContain("had already read");
expect(note).toContain("edited these messages after that run read them");
expect(note).toContain("FOLLOWUP-NEW");
}, 60_000);
it.each([
{ title: "a pending approval", setup: "approval" as const, errorCode: "native_continuation_requires_reconciliation" },
{ title: "a credential the person has not shared", setup: "none" as const, errorCode: "configuration_incomplete" },
@@ -24,6 +24,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { readRunCancellation } from "./run-cancellation.js";
import { readSystemRecovery } from "./system-recovery-state.js";
import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js";
/**
* Fresh-continuation fallback: exact resumption of an unfinished run is an
@@ -196,13 +197,15 @@ async function unverifiedActions(tx: Db, run: typeof heartbeatRuns.$inferSelect)
return [...names].slice(0, 20);
}
export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[] };
export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[];
/** Messages the run read that a person edited afterwards: the run never saw the current text. */
editedFollowUps?: string[] };
/** Comment ids the run read through get_task_history, with the read time (the
* queued-comment owner's `task_history.read` reader). */
/** Comment ids the run read through get_task_history, with the read time and
* the comment revision it read (the queued-comment owner's `task_history.read` reader). */
async function taskHistoryReadsOfRun(tx: Db, run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">) {
const reads = await taskHistoryReadsForRun(tx, { companyId: run.companyId, runId: run.id });
return new Map(reads.map(read => [read.commentId, read.readAt]));
return new Map(reads.map(read => [read.commentId, read]));
}
const quoted = (value: string, max = 80) => `"${value.replace(/\s+/g, " ").trim().slice(0, max)}${value.length > max ? "..." : ""}"`;
@@ -222,11 +225,18 @@ async function createdObjects(tx: Db, run: typeof heartbeatRuns.$inferSelect): P
.where(and(eq(documentRevisions.companyId, run.companyId), eq(documentRevisions.createdByRunId, run.id))).limit(10),
]);
const reads = await taskHistoryReadsOfRun(tx, run);
const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]),
eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : [];
const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body, updatedAt: issueComments.updatedAt })
.from(issueComments).where(and(
eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]),
eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : [];
// Only the revision the run read counts as read: a message edited since then
// is quoted as changed, so the agent treats its current text as new.
const unchanged = (comment: (typeof readComments)[number]) => comment.updatedAt.getTime() <= reads.get(comment.id)!.revision.getTime();
const edited = readComments.filter(comment => !unchanged(comment));
return {
readFollowUps: readComments.map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.toISOString().slice(11, 16)} UTC)`),
readFollowUps: readComments.filter(unchanged)
.map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.readAt.toISOString().slice(11, 16)} UTC)`),
...(edited.length ? { editedFollowUps: edited.map(comment => quoted(comment.body)) } : {}),
subtasks: children.map(child => `${child.identifier ?? "subtask"} ${quoted(child.title)}`),
comments: comments.map(comment => quoted(comment.body)),
documents: revisions.map(revision => quoted(revision.title ?? "untitled document")),
@@ -252,14 +262,17 @@ export function freshContinuationNote(input: { runId: string; unverified: string
const read = created?.readFollowUps?.length
? ` That run had already read these messages from a person: ${created.readFollowUps.join(", ")}. Answer each once; what it did after reading is below.`
: "";
const existing = created && (created.subtasks.length || created.comments.length || created.documents.length)
const edited = created?.editedFollowUps?.length
? ` A person edited these messages after that run read them, so it never saw their current text; treat them as new: ${created.editedFollowUps.join(", ")}.`
: "";
const existing = created &&(created.subtasks.length || created.comments.length || created.documents.length)
? ` That run already did these, and they exist now: ${[
created.subtasks.length ? `created subtasks ${created.subtasks.join(", ")}` : null,
created.comments.length ? `posted comments ${created.comments.join(", ")}` : null,
created.documents.length ? `saved documents ${created.documents.join(", ")}` : null,
].filter(Boolean).join("; ")}. Do not create or post them again; read them and continue from where they leave off.`
: "";
return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${existing} ${actions} ${guidance}`;
return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${edited}${existing} ${actions} ${guidance}`;
}
const USER_TEXT = "The previous run was cut off. Paperclip continues the task in a fresh run from its last saved work.";
@@ -874,12 +887,19 @@ export async function settleCoalescedFreshContinuationHolds(db: Db, input: { com
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'coalesced'`));
}
/** A fresh continuation of this task is recorded and its run does not exist yet. */
export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string) {
/** A fresh continuation of this task is recorded, its run does not exist yet,
* and delivery still owns it. Past PENDING_CONTINUATION_BOUND_MS delivery has
* had its chance (it re-drives a record held past its hold bound first), so the
* record no longer absorbs a repair wake: that wake becomes the task's one
* successor, and delivery then settles the record as delivered by it. */
export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string, now = new Date()) {
const [row] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.status, "resolved"),
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
${issueRecoveryActions.updatedAt}::text)::timestamptz
> ${new Date(now.getTime() - PENDING_CONTINUATION_BOUND_MS).toISOString()}::timestamptz`,
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`)).limit(1);
return Boolean(row);
}
@@ -0,0 +1,17 @@
/**
* How long a recorded recovery continuation may wait for delivery. One module
* owns both numbers so delivery, wake admission and the assigned-work
* invariant agree on when a pending continuation stops being the task's path.
*/
/** Delivery pacing holds a continuation back at most this long: past it the
* delivery worker admits it even when the agent's queue is full or another
* recovery run of the agent is queued. */
export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000;
/** A continuation still pending past this bound (the hold bound plus 2 minutes
* of 15-second delivery passes) is not going to be delivered: the invariant no
* longer counts it as pending, and wake admission no longer coalesces a repair
* wake into it, so that wake becomes the task's one successor. Delivery
* re-drives first, so within the bound there is exactly one owner. */
export const PENDING_CONTINUATION_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS + 2 * 60_000;
@@ -29,6 +29,7 @@ import { issueTreeControlService } from "../issue-tree-control.js";
import { taskInterruptionPresentations } from "../task-interruption-presentation.js";
import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";
import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";
import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-bounds.js";
import {
continuationActivityMessage,
continuationAllowance,
@@ -59,10 +60,9 @@ const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
/** How long a deferred wake (no run holding its task) or a claimed wake with no run counts as pending. */
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
/** How long a held continuation (paced delivery) counts as pending: the pacing's
* admission bound (RECOVERY_CONTINUATION_MAX_HOLD_MS, 5 min) plus 2 minutes of
* 15-second delivery passes. */
export const PENDING_CONTINUATION_BOUND_MS = 7 * 60_000;
/** How long a held continuation (paced delivery) counts as pending; shared with
* wake admission and delivery. */
export { PENDING_CONTINUATION_BOUND_MS };
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];