mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(recovery): repair a continuation whose delivery is stuck; an edited follow-up is not shown as read
- One module (recovery-continuation-bounds.ts) owns the hold bound (5 min) and the pending bound (hold bound + 2 min); the invariant, wake admission and delivery read both from it. - Delivery: a continuation held past the hold bound is no longer skipped because another recovery run of its agent is queued. - Admission: hasPendingFreshContinuation counts a record only within the pending bound. Past it a repair wake becomes the task's run; delivery's newer-run check settles the record as delivered by it (one successor). - Opening note: only the comment revision the cut-off run read counts as read; a follow-up edited afterwards is listed as changed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cfd42b0516
commit
b76111e45f
6 files changed
+119
-19
No files matched your search
@@ -299,4 +299,19 @@ suite("recovery wake herd", () => {
|
||||
const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
|
||||
expect(stale.pendingWakes).toBe(facts.pendingWakes - 1);
|
||||
}, 60_000);
|
||||
|
||||
it("re-drives a continuation held past the hold bound even while another recovery run of its agent stays queued", async () => {
|
||||
const agentId = await seedCompany(1);
|
||||
await queuedRecoveryRun(agentId, 1);
|
||||
const held = await pendingContinuation(agentId, 30);
|
||||
const wake = fakeWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).not.toHaveBeenCalled();
|
||||
const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString();
|
||||
await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
|
||||
'{freshContinuation,recordedAt}', to_jsonb(${heldSince}::text))` }).where(eq(issueRecoveryActions.id, held.action.id));
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect(await deliveryOf(held.action.id)).toBe("delivered");
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -42,9 +42,10 @@ import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCh
|
||||
import { alias } from "drizzle-orm/pg-core";
|
||||
import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js";
|
||||
import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js";
|
||||
import { RECOVERY_CONTINUATION_MAX_HOLD_MS } from "./recovery-continuation-bounds.js";
|
||||
|
||||
export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5;
|
||||
export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000;
|
||||
export { RECOVERY_CONTINUATION_MAX_HOLD_MS };
|
||||
const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8;
|
||||
|
||||
/** An operator records observed outcomes; this is not permission to blindly retry. */
|
||||
@@ -389,11 +390,15 @@ export async function deliverReconciledExecutions(
|
||||
seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined,
|
||||
seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined,
|
||||
companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined,
|
||||
sql`not exists (select 1 from ${heartbeatRuns} queued_recovery
|
||||
// One queued recovery run per agent paces the rest, but only within
|
||||
// the hold bound: a continuation held past it is admitted like one
|
||||
// held behind ordinary work (agentHasRoom), or it would wait as long
|
||||
// as that other run stays queued.
|
||||
sql`(${heldSince} <= ${heldBound}::timestamptz or not exists (select 1 from ${heartbeatRuns} queued_recovery
|
||||
where queued_recovery.company_id = ${issueRecoveryActions.companyId}
|
||||
and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId}
|
||||
and queued_recovery.status = 'queued'
|
||||
and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)}))`,
|
||||
and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`,
|
||||
),
|
||||
)
|
||||
.orderBy(
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js";
|
||||
import { getExecutionBlocker } from "./execution-blocker.js";
|
||||
import {
|
||||
FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation,
|
||||
@@ -315,6 +316,30 @@ suite("fresh-continuation fallback", () => {
|
||||
expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered");
|
||||
}, 60_000);
|
||||
|
||||
it("lets a repair wake become the one successor once a pending continuation is past the bound, and coalesces it within the bound", async () => {
|
||||
const f = await unfinishedRun({ runtime: "native", errorCode: "native_continuation_requires_reconciliation",
|
||||
error: "runner_remote_process_identity_unavailable", hold: "settled_blocked" });
|
||||
await occupySlot();
|
||||
expect(await sweepWake(f.issueId)).toBeNull();
|
||||
const record = (await pendingRecord(f.issueId))!;
|
||||
expect(record.evidence.continuationDelivery).toBe("pending");
|
||||
const taskRuns = async () => db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`));
|
||||
// Within the bound, the repair wake is coalesced into the pending record.
|
||||
expect(await sweepWake(f.issueId)).toBeNull();
|
||||
expect(await taskRuns()).toHaveLength(0);
|
||||
// Delivery is stuck past the bound: the next repair wake becomes the continuation.
|
||||
const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString();
|
||||
await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
|
||||
'{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, record.id));
|
||||
expect(await sweepWake(f.issueId)).not.toBeNull();
|
||||
expect(await taskRuns()).toHaveLength(1);
|
||||
// Delivery then settles the record as delivered by that run; still exactly one run.
|
||||
await deliver();
|
||||
expect(await taskRuns()).toHaveLength(1);
|
||||
expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered");
|
||||
}, 60_000);
|
||||
|
||||
// A gate after the hook must never consume the recovery: it stays pending
|
||||
// and is delivered once the gate clears.
|
||||
it("keeps the continuation pending while the agent's daily run limit declines it, then delivers it", async () => {
|
||||
@@ -826,6 +851,24 @@ suite("fresh-continuation fallback", () => {
|
||||
expect(note.indexOf("had already read")).toBeLessThan(note.indexOf("DONE-0d981334"));
|
||||
}, 60_000);
|
||||
|
||||
it("does not list a follow-up edited after the cut-off run read it as already read; it is marked as changed", async () => {
|
||||
const f = await unfinishedRun({ runtime: "native", errorCode: "native_restart_recovery_blocked" });
|
||||
const [followUp] = await db.insert(issueComments).values({ companyId, issueId: f.issueId, authorType: "user",
|
||||
authorUserId: "board-user", body: "Additional requirement: include the line FOLLOWUP-OLD" } as typeof issueComments.$inferInsert).returning();
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId: f.runId, agentId, seq: 1, eventType: "task_history.read",
|
||||
stream: "system", level: "info", payload: { comments: { [followUp!.id]: followUp!.updatedAt.toISOString() } },
|
||||
} as typeof heartbeatRunEvents.$inferInsert);
|
||||
// The person edits the follow-up after the run read it.
|
||||
await db.update(issueComments).set({ body: "Additional requirement: include the line FOLLOWUP-NEW",
|
||||
updatedAt: new Date(followUp!.updatedAt.getTime() + 60_000) }).where(eq(issueComments.id, followUp!.id));
|
||||
expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })).toMatchObject({ kind: "continued" });
|
||||
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!));
|
||||
const note = (hold!.evidence.executionReconciliation as Record<string, string>).outcomeEvidence;
|
||||
expect(note).not.toContain("had already read");
|
||||
expect(note).toContain("edited these messages after that run read them");
|
||||
expect(note).toContain("FOLLOWUP-NEW");
|
||||
}, 60_000);
|
||||
|
||||
it.each([
|
||||
{ title: "a pending approval", setup: "approval" as const, errorCode: "native_continuation_requires_reconciliation" },
|
||||
{ title: "a credential the person has not shared", setup: "none" as const, errorCode: "configuration_incomplete" },
|
||||
|
||||
@@ -24,6 +24,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { readRunCancellation } from "./run-cancellation.js";
|
||||
import { readSystemRecovery } from "./system-recovery-state.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS } from "./recovery-continuation-bounds.js";
|
||||
|
||||
/**
|
||||
* Fresh-continuation fallback: exact resumption of an unfinished run is an
|
||||
@@ -196,13 +197,15 @@ async function unverifiedActions(tx: Db, run: typeof heartbeatRuns.$inferSelect)
|
||||
return [...names].slice(0, 20);
|
||||
}
|
||||
|
||||
export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[] };
|
||||
export type FreshContinuationCreatedObjects = { subtasks: string[]; comments: string[]; documents: string[]; readFollowUps?: string[];
|
||||
/** Messages the run read that a person edited afterwards: the run never saw the current text. */
|
||||
editedFollowUps?: string[] };
|
||||
|
||||
/** Comment ids the run read through get_task_history, with the read time (the
|
||||
* queued-comment owner's `task_history.read` reader). */
|
||||
/** Comment ids the run read through get_task_history, with the read time and
|
||||
* the comment revision it read (the queued-comment owner's `task_history.read` reader). */
|
||||
async function taskHistoryReadsOfRun(tx: Db, run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">) {
|
||||
const reads = await taskHistoryReadsForRun(tx, { companyId: run.companyId, runId: run.id });
|
||||
return new Map(reads.map(read => [read.commentId, read.readAt]));
|
||||
return new Map(reads.map(read => [read.commentId, read]));
|
||||
}
|
||||
|
||||
const quoted = (value: string, max = 80) => `"${value.replace(/\s+/g, " ").trim().slice(0, max)}${value.length > max ? "..." : ""}"`;
|
||||
@@ -222,11 +225,18 @@ async function createdObjects(tx: Db, run: typeof heartbeatRuns.$inferSelect): P
|
||||
.where(and(eq(documentRevisions.companyId, run.companyId), eq(documentRevisions.createdByRunId, run.id))).limit(10),
|
||||
]);
|
||||
const reads = await taskHistoryReadsOfRun(tx, run);
|
||||
const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
|
||||
eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]),
|
||||
eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : [];
|
||||
const readComments = reads.size ? await tx.select({ id: issueComments.id, body: issueComments.body, updatedAt: issueComments.updatedAt })
|
||||
.from(issueComments).where(and(
|
||||
eq(issueComments.companyId, run.companyId), inArray(issueComments.id, [...reads.keys()]),
|
||||
eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt))).limit(10) : [];
|
||||
// Only the revision the run read counts as read: a message edited since then
|
||||
// is quoted as changed, so the agent treats its current text as new.
|
||||
const unchanged = (comment: (typeof readComments)[number]) => comment.updatedAt.getTime() <= reads.get(comment.id)!.revision.getTime();
|
||||
const edited = readComments.filter(comment => !unchanged(comment));
|
||||
return {
|
||||
readFollowUps: readComments.map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.toISOString().slice(11, 16)} UTC)`),
|
||||
readFollowUps: readComments.filter(unchanged)
|
||||
.map(comment => `${quoted(comment.body)} (read at ${reads.get(comment.id)!.readAt.toISOString().slice(11, 16)} UTC)`),
|
||||
...(edited.length ? { editedFollowUps: edited.map(comment => quoted(comment.body)) } : {}),
|
||||
subtasks: children.map(child => `${child.identifier ?? "subtask"} ${quoted(child.title)}`),
|
||||
comments: comments.map(comment => quoted(comment.body)),
|
||||
documents: revisions.map(revision => quoted(revision.title ?? "untitled document")),
|
||||
@@ -252,14 +262,17 @@ export function freshContinuationNote(input: { runId: string; unverified: string
|
||||
const read = created?.readFollowUps?.length
|
||||
? ` That run had already read these messages from a person: ${created.readFollowUps.join(", ")}. Answer each once; what it did after reading is below.`
|
||||
: "";
|
||||
const existing = created && (created.subtasks.length || created.comments.length || created.documents.length)
|
||||
const edited = created?.editedFollowUps?.length
|
||||
? ` A person edited these messages after that run read them, so it never saw their current text; treat them as new: ${created.editedFollowUps.join(", ")}.`
|
||||
: "";
|
||||
const existing = created &&(created.subtasks.length || created.comments.length || created.documents.length)
|
||||
? ` That run already did these, and they exist now: ${[
|
||||
created.subtasks.length ? `created subtasks ${created.subtasks.join(", ")}` : null,
|
||||
created.comments.length ? `posted comments ${created.comments.join(", ")}` : null,
|
||||
created.documents.length ? `saved documents ${created.documents.join(", ")}` : null,
|
||||
].filter(Boolean).join("; ")}. Do not create or post them again; read them and continue from where they leave off.`
|
||||
: "";
|
||||
return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${existing} ${actions} ${guidance}`;
|
||||
return `The previous attempt (run ${input.runId}) was cut off before it finished. ${files}${read}${edited}${existing} ${actions} ${guidance}`;
|
||||
}
|
||||
|
||||
const USER_TEXT = "The previous run was cut off. Paperclip continues the task in a fresh run from its last saved work.";
|
||||
@@ -874,12 +887,19 @@ export async function settleCoalescedFreshContinuationHolds(db: Db, input: { com
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'coalesced'`));
|
||||
}
|
||||
|
||||
/** A fresh continuation of this task is recorded and its run does not exist yet. */
|
||||
export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string) {
|
||||
/** A fresh continuation of this task is recorded, its run does not exist yet,
|
||||
* and delivery still owns it. Past PENDING_CONTINUATION_BOUND_MS delivery has
|
||||
* had its chance (it re-drives a record held past its hold bound first), so the
|
||||
* record no longer absorbs a repair wake: that wake becomes the task's one
|
||||
* successor, and delivery then settles the record as delivered by it. */
|
||||
export async function hasPendingFreshContinuation(db: Db, companyId: string, issueId: string, now = new Date()) {
|
||||
const [row] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
|
||||
sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
|
||||
${issueRecoveryActions.updatedAt}::text)::timestamptz
|
||||
> ${new Date(now.getTime() - PENDING_CONTINUATION_BOUND_MS).toISOString()}::timestamptz`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`)).limit(1);
|
||||
return Boolean(row);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* How long a recorded recovery continuation may wait for delivery. One module
|
||||
* owns both numbers so delivery, wake admission and the assigned-work
|
||||
* invariant agree on when a pending continuation stops being the task's path.
|
||||
*/
|
||||
|
||||
/** Delivery pacing holds a continuation back at most this long: past it the
|
||||
* delivery worker admits it even when the agent's queue is full or another
|
||||
* recovery run of the agent is queued. */
|
||||
export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000;
|
||||
|
||||
/** A continuation still pending past this bound (the hold bound plus 2 minutes
|
||||
* of 15-second delivery passes) is not going to be delivered: the invariant no
|
||||
* longer counts it as pending, and wake admission no longer coalesces a repair
|
||||
* wake into it, so that wake becomes the task's one successor. Delivery
|
||||
* re-drives first, so within the bound there is exactly one owner. */
|
||||
export const PENDING_CONTINUATION_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS + 2 * 60_000;
|
||||
@@ -29,6 +29,7 @@ import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { taskInterruptionPresentations } from "../task-interruption-presentation.js";
|
||||
import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";
|
||||
import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-bounds.js";
|
||||
import {
|
||||
continuationActivityMessage,
|
||||
continuationAllowance,
|
||||
@@ -59,10 +60,9 @@ const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
|
||||
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
|
||||
/** How long a deferred wake (no run holding its task) or a claimed wake with no run counts as pending. */
|
||||
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
|
||||
/** How long a held continuation (paced delivery) counts as pending: the pacing's
|
||||
* admission bound (RECOVERY_CONTINUATION_MAX_HOLD_MS, 5 min) plus 2 minutes of
|
||||
* 15-second delivery passes. */
|
||||
export const PENDING_CONTINUATION_BOUND_MS = 7 * 60_000;
|
||||
/** How long a held continuation (paced delivery) counts as pending; shared with
|
||||
* wake admission and delivery. */
|
||||
export { PENDING_CONTINUATION_BOUND_MS };
|
||||
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
||||
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
|
||||
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];
|
||||
|
||||
Reference in new issue
Block a user