fix(recovery): continue a task whose system retry was refused at claim behind a saved-work hold

A follow-up run that failed before provider work left a saved-work hold.
The dispatch gate cancelled the system's own retry, and the fallback then
left the task for a Retry that was not offered. A system recovery run
refused at dispatch now counts as the system continuing the task. Attempts
that never started do not spend the hourly budget. The sweep continues a
blocked task behind a refused run. Notices never ask for a Retry the task
does not offer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-13 15:37:01 -06:00
1 parent 2a8b485140
commit ce4431410b
9 files changed
+235 -11

No files matched your search

@@ -414,4 +414,36 @@ describePostgres("a continuation that fails before doing any work is continued a
expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" });
expect(await continuationsOf(stopped.id)).toHaveLength(1);
}, 60_000);
it("the sweep continues a blocked task whose system retry was refused at claim behind a recovered saved-work hold", async () => {
resetBacklogContinuationPacingForTests(db);
const f = await seed();
const blockedAt = new Date(Date.now() - 20 * MINUTE);
const task = await f.task({ status: "blocked", blockedTransitionAt: blockedAt, statusVersion: 3 });
const source = await f.run(task.id, { errorCode: "adapter_failed", error: "native_instruction_materialization_handoff_unavailable",
finishedAt: new Date(blockedAt.getTime() - 500), resultJson: { workspaceRestoreFailure: "restore_failed" },
contextSnapshot: { wakeReason: "issue_children_completed" } });
const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog",
status: "active", ownerType: "board", returnOwnerAgentId: f.agentId, cause: "legacy_execution_requires_reconciliation",
fingerprint: `legacy-execution:${source.id}`, nextAction: "Held.", createdAt: blockedAt,
evidence: { runId: source.id, originalFailureCode: "adapter_failed", workspaceRestoreFailure: "restore_failed",
systemRecovery: { ...createSystemRecovery(source.id), state: "complete" } } }).returning();
const refused = await f.run(task.id, { status: "cancelled", errorCode: "execution_reconciliation_required", startedAt: null,
retryOfRunId: source.id, finishedAt: new Date(blockedAt.getTime() + 1_000), createdAt: new Date(blockedAt.getTime() - 100), resultJson: {},
contextSnapshot: { wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed" } });
const queue = wakeQueue();
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(task.id);
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }))
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } });
const [record] = await continuationsOf(source.id);
expect(record?.id).toBe(hold!.id);
expect(record?.evidence).toMatchObject({ continuationDelivery: "pending", freshContinuation: { sourceRunId: source.id, trigger: "assigned_work" },
freshContinuationSettlement: { refusedAtDispatchRunId: refused.id, requestedBy: "system" } });
expect((await taskRow(task.id)).status).toBe("todo");
expect(queue.calls).toHaveLength(0);
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }))
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent" } });
}, 60_000);
});
@@ -29,6 +29,8 @@ import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapte
import { createSystemRecovery } from "./system-recovery-state.js";
import { requestTaskRetry } from "./task-retry.js";
import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js";
import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js";
import { taskAutomaticAttempts } from "./recovery/task-automatic-attempts.js";
const support = await getEmbeddedPostgresTestSupport();
const suite = support.supported ? describe : describe.skip;
@@ -1323,4 +1325,125 @@ suite("fresh-continuation fallback", () => {
const [next] = await successors(f.runId);
expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" });
}, 60_000);
async function followUpFailedBeforeProviderWork(options: { restoreFailed?: boolean } = {}) {
const issueId = randomUUID(), runId = randomUUID();
const endedAt = new Date(Date.now() - 60_000);
await db.insert(issues).values({ id: issueId, companyId, title: "Summarize the subtasks", status: "in_progress", assigneeAgentId: agentId });
const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, mode: "isolated",
strategyType: "git_worktree", name: "Task workspace", cwd: "/work/remote-task" }).returning();
await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issueId));
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "automation", issueId,
runtimeMode: "legacy", status: "running", startedAt: new Date(endedAt.getTime() - 10_000),
createdAt: new Date(endedAt.getTime() - 60_000), responsibleUserId: userId,
contextSnapshot: { issueId, executionWorkspaceId: workspace!.id, wakeReason: "issue_children_completed" } }).returning();
const [environment] = await db.insert(environments).values({ name: `sandbox-${runId}`, driver: "sandbox" }).returning();
await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace!.id,
heartbeatRunId: runId, environmentId: environment!.id, status: "active", leasePolicy: "reuse_by_environment", provider: "daytona",
providerLeaseId: `sandbox-${runId}`, metadata: { remoteCwd: "/work/remote-task", driver: "sandbox",
workspaceRealization: { authoritativeRoot: "/work/remote-task" } } });
await terminalizeLegacyExecution({ db, run: run!, status: "failed", patch: { errorCode: "adapter_failed",
error: "native_instruction_materialization_handoff_unavailable", finishedAt: endedAt,
resultJson: options.restoreFailed === false ? { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }
: { workspaceRestoreFailure: "restore_failed" } } });
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
return { issueId, runId, actionId: hold?.id ?? null };
}
async function savedWorkRecoveryState(actionId: string, runId: string, state: "running" | "complete") {
await db.update(issueRecoveryActions).set({ updatedAt: new Date(),
evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ systemRecovery: { ...createSystemRecovery(runId), state } })}::jsonb` })
.where(eq(issueRecoveryActions.id, actionId));
}
async function systemRetryAtClaim(f: { issueId: string; runId: string }) {
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId,
runtimeMode: "legacy", status: "queued", retryOfRunId: f.runId,
contextSnapshot: { issueId: f.issueId, wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed",
retryOfRunId: f.runId } }).returning();
const outcome = await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: run!.id,
expectedStatus: "queued", now: new Date() });
return { run: run!, outcome };
}
async function deliveredContinuationIsClaimable(f: { issueId: string; runId: string; actionId: string | null }, refusedRunId: string) {
await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId, issueId: f.issueId });
expect((await issue(f.issueId)).status).not.toBe("blocked");
expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull();
await occupySlot();
await deliver();
const next = (await successors(f.runId)).filter(run => run.id !== refusedRunId);
expect(next).toHaveLength(1);
expect(next[0]!.status).toBe("queued");
expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: next[0]!.id,
expectedStatus: "queued", now: new Date() })).toEqual({ outcome: "not_stale" });
return next[0]!;
}
it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered before the system retry was refused at claim", async () => {
const f = await followUpFailedBeforeProviderWork();
expect(f.actionId).not.toBeNull();
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
const refused = await systemRetryAtClaim(f);
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
expect((await holdRow(f.actionId!)).evidence).toMatchObject({
freshContinuation: { sourceRunId: f.runId }, continuationDelivery: "pending",
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
await deliveredContinuationIsClaimable(f, refused.run.id);
}, 60_000);
it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered after the system retry was refused at claim", async () => {
const f = await followUpFailedBeforeProviderWork();
await savedWorkRecoveryState(f.actionId!, f.runId, "running");
const refused = await systemRetryAtClaim(f);
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
expect((await holdRow(f.actionId!)).evidence.freshContinuationCheck)
.toMatchObject({ kind: "exact_recovery_pending", reason: "saved_work_recovery_running" });
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
await makeDue(f.actionId!);
expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 });
expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId },
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
await deliveredContinuationIsClaimable(f, refused.run.id);
}, 60_000);
it("continues a follow-up that failed before provider work with its files in place, after its system retry was refused at claim", async () => {
const f = await followUpFailedBeforeProviderWork({ restoreFailed: false });
const refused = await systemRetryAtClaim(f);
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId },
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
await deliveredContinuationIsClaimable(f, refused.run.id);
}, 60_000);
it("counts only automatic retries that started, or still may, against the task's hourly budget", async () => {
const f = await followUpFailedBeforeProviderWork();
for (let index = 0; index < FRESH_CONTINUATIONS_PER_HOUR; index++) {
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
status: "cancelled", errorCode: "execution_reconciliation_required", finishedAt: new Date(), retryOfRunId: f.runId,
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
}
expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(0);
expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(0);
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
status: "failed", errorCode: "adapter_failed", startedAt: new Date(), finishedAt: new Date(), retryOfRunId: f.runId,
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
status: "scheduled_retry", retryOfRunId: f.runId,
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(2);
expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(2);
}, 60_000);
it("still leaves a person's failed Retry of a follow-up for that person after its successor is refused at claim", async () => {
const f = await followUpFailedBeforeProviderWork();
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
await db.update(heartbeatRuns).set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` })
.where(eq(heartbeatRuns.id, f.runId));
const refused = await systemRetryAtClaim(f);
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
expect((await holdRow(f.actionId!)).evidence.freshContinuation).toBeUndefined();
expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" }))
.toEqual({ kind: "person_owned", reason: "retry_needed" });
}, 60_000);
});
@@ -26,7 +26,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { readRunCancellation } from "./run-cancellation.js";
import { readSystemRecovery } from "./system-recovery-state.js";
import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuesAutomaticallySql, isPersonOwnedFailureCode } from "./continuation-cause.js";
import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js";
import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js";
export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js";
@@ -813,7 +813,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ
and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at)
or exists (select 1 from ${heartbeatRuns} held where ${refusedLater}
and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`);
const recent = source(sql`${continuesAutomaticallySql("source")}
const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")}
and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false)
and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`;
const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource})
and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`);
return { recent, personAsked };
}
@@ -466,6 +466,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
* Chat conversations and blocks older than the stale window are left out:
* the first wait on a person's message, the second is only reported.
*/
function latestRunRefusedAtDispatch(at: Date) {
return sql`coalesce((select lr.status = 'cancelled' and lr.started_at is null and lr.error_code = 'execution_reconciliation_required'
and lr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz
from heartbeat_runs lr where lr.company_id = ${issues.companyId} and (lr.context_snapshot->>'issueId') = ${issues.id}::text
order by lr.created_at desc limit 1), false)`;
}
async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) {
const at = now();
const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
@@ -518,7 +525,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
-- except a recent board escalation, which the predicate judges by its failed run's cause
and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr
where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId'
and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)))`,
and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz))
and not (${issues.status} = 'blocked' and ra.cause not in ('configuration_incomplete', 'ownership_owner_unprovable')
and ${latestRunRefusedAtDispatch(at)}))`,
// An assignee that cannot take work (paused, being set up, held) is
// its own wait; the predicate names it when anything else asks.
sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId}
@@ -563,6 +572,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
// hold's owner (the fresh-continuation fallback, same per-task cap). The
// sweep reports it and repairs only blocks that have no hold at all, so
if (issue.status === "blocked" && facts.activeRecoveryAction && !facts.activeRecoveryAction.continuedBySweep) {
const refused = facts.latestRun?.status === "cancelled" && facts.latestRun.errorCode === "execution_reconciliation_required"
? await continueFromSavedWork(issue, facts, at) : null;
if (refused?.kind === "continued" || refused?.kind === "already_continued") return { verdict, outcome: "repaired", retryAt: null };
if (refused?.kind === "backoff" || refused?.kind === "exact_recovery_pending")
return { verdict, outcome: refused.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: refused.retryAt };
return { verdict, outcome: "left_to_hold_owner", retryAt: null };
}
// A stalled native run is ended by native recovery, not here: the sweep
@@ -624,8 +638,21 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise<FreshContinuationResult | null> {
const source = facts.latestRun;
if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)
|| (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required")) return null;
if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)) return null;
if (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required") {
const [refusedFor] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, issue.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`,
inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), sql`${heartbeatRuns.finishedAt} is not null`,
sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null
and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`))
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
if (!refusedFor) return null;
return ensureFreshContinuation(db, { companyId: issue.companyId, runId: refusedFor.id, trigger: "assigned_work", now: at })
.catch(error => {
logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation behind a refused run deferred");
return null;
});
}
const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error,
retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns)
@@ -6,6 +6,11 @@ export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure";
export const AUTOMATIC_RETRY_REASONS = [SCHEDULER_TRANSIENT_RETRY_REASON, "server_restart", "interrupted_preparation"];
export const SHARED_CONTINUATION_WINDOW_MS = 60 * 60_000;
export function startedOrPendingAutomaticRetry() {
return sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null
and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`;
}
export async function freshContinuationTimes(db: Db, companyId: string, issueId: string, since: Date) {
const rows = await db.select({ recordedAt: sql<string>`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`,
trigger: sql<string | null>`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}'` }).from(issueRecoveryActions).where(and(
@@ -25,7 +30,8 @@ export async function sharedContinuationAttempts(db: Db, companyId: string, issu
sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`)).limit(64),
db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))).limit(64),
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since),
startedOrPendingAutomaticRetry())).limit(64),
freshContinuationTimes(db, companyId, issueId, since),
]);
return [...plainContinuations.map(row => row.createdAt), ...retries.map(row => row.createdAt), ...fresh.map(record => record.at)]
+10
View File
@@ -176,6 +176,7 @@ import {
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildExecutionReviewParticipantUnavailableNoticeSeed,
buildStrandedRecoveryEscalationNotice,
withoutRetryInstruction,
buildNoRunningExecutionNoticeSeed,
RETRY_FROM_SAVED_WORK_NEXT_ACTION,
type StrandedRecoveryNoticeSeed,
@@ -185,6 +186,7 @@ import {
isStrandedIssueRecoveryOriginKind,
} from "./origins.js";
import { withRecoveryContext } from "./status-only-context.js";
import { getExecutionBlocker } from "../execution-blocker.js";
import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js";
import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js";
import { rehomeStrandedLegacyWorkspaceTasks } from "./legacy-workspace-rehome.js";
@@ -4250,6 +4252,13 @@ export function recoveryService(
recoveryCause,
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
}, tx);
const executionBlocker = await getExecutionBlocker(tx as unknown as Db, input.issue.companyId, input.issue.id);
const retryOffered = !(executionBlocker && executionBlocker.canRetry === false);
if (!retryOffered && /\bretry\b/i.test(recoveryAction.nextAction ?? "")) {
recoveryAction.nextAction = withoutRetryInstruction(recoveryAction.nextAction!);
await tx.update(issueRecoveryActions).set({ nextAction: recoveryAction.nextAction })
.where(and(eq(issueRecoveryActions.companyId, input.issue.companyId), eq(issueRecoveryActions.id, recoveryAction.id)));
}
const isProviderQuotaWait =
recoveryCause === "provider_quota" &&
!recoveryAction.ownerAgentId &&
@@ -4333,6 +4342,7 @@ export function recoveryService(
fallbackBody: input.comment,
recoveryCause,
recoveryActionId: recoveryAction.id,
retryOffered,
continuesAutomatically: Boolean(input.latestRun) && recoveryAction.kind === STRANDED_KIND
&& recoveryAction.ownerType === "board" && input.latestRun?.agentId === input.issue.assigneeAgentId
&& strandedRunCauseIsTemporary({
@@ -401,3 +401,16 @@ it("tells the operator to switch a preconfigured Runner home to an isolated one"
expect(notice.nextAction).toContain("isolated Runner home");
expect(notice.nextAction).not.toContain("Choose an AI connection");
});
it("never tells a person to press Retry when the task offers no Retry", () => {
const sourceRun = { id: "run-1", status: "cancelled", errorCode: "execution_reconciliation_required" };
for (const seed of [buildNoRunningExecutionNoticeSeed({ status: "in_progress" }), null]) {
const offered = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun });
expect(`${offered.body} ${JSON.stringify(allRows(offered.metadata))}`).toMatch(/Press Retry/);
const withheld = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun,
retryOffered: false });
expect(withheld.body).not.toMatch(/retry/i);
expect(JSON.stringify(allRows(withheld.metadata))).not.toMatch(/retry/i);
expect(withheld.body).toContain("Paperclip is holding this task until its saved work is settled.");
}
});
@@ -27,6 +27,13 @@ export type StrandedRecoveryNoticeSeed = {
nextAction?: string;
};
export const SAVED_WORK_HELD_SENTENCE = "Paperclip is holding this task until its saved work is settled.";
export function withoutRetryInstruction(text: string): string {
const kept = text.split(/(?<=[.!?])\s+/).filter(sentence => !/\bretry\b/i.test(sentence));
return [...kept, SAVED_WORK_HELD_SENTENCE].join(" ");
}
export type StrandedRecoveryEscalationNotice = {
body: string;
presentation: IssueCommentPresentation;
@@ -270,6 +277,7 @@ export function buildStrandedRecoveryEscalationNotice(input: {
recoveryCause?: string | null;
recoveryActionId: string;
continuesAutomatically?: boolean;
retryOffered?: boolean;
recoveryOwner: { id: string; name: string | null } | null | undefined;
sourceRun: {
id: string;
@@ -292,7 +300,8 @@ export function buildStrandedRecoveryEscalationNotice(input: {
tone: "danger" as const,
} : SETUP_FAILURE_NOTICES[input.sourceRun?.errorCode ?? ""] ?? input.seed;
const fallbackBody = input.fallbackBody?.trim();
const body = seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY);
const honest = (text: string) => input.retryOffered === false ? withoutRetryInstruction(text) : text;
const body = honest(seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY));
const title =
STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE[input.sourceRun?.errorCode?.trim() ?? ""] ??
seed?.title ??
@@ -310,9 +319,9 @@ export function buildStrandedRecoveryEscalationNotice(input: {
),
keyValueRow(
"Next action",
seed?.nextAction ?? (input.recoveryOwner
honest(seed?.nextAction ?? (input.recoveryOwner
? "The assigned agent continues the task from the saved work"
: RETRY_FROM_SAVED_WORK_NEXT_ACTION),
: RETRY_FROM_SAVED_WORK_NEXT_ACTION)),
),
];
@@ -1,6 +1,6 @@
import { and, desc, eq, gte, inArray, sql } from "drizzle-orm";
import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db";
import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes } from "./continuation-ledger.js";
import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes, startedOrPendingAutomaticRetry } from "./continuation-ledger.js";
import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js";
export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON };
@@ -14,7 +14,8 @@ export async function taskAutomaticAttempts(db: Db, companyId: string, issueId:
.orderBy(desc(activityLog.createdAt)).limit(64),
db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since)))
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since),
startedOrPendingAutomaticRetry()))
.orderBy(desc(heartbeatRuns.createdAt)).limit(64),
]);
const fresh = (await freshContinuationTimes(db, companyId, issueId, since))