mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
fix(recovery): continue a task whose system retry was refused at claim behind a saved-work hold
A follow-up run that failed before provider work left a saved-work hold. The dispatch gate cancelled the system's own retry, and the fallback then left the task for a Retry that was not offered. A system recovery run refused at dispatch now counts as the system continuing the task. Attempts that never started do not spend the hourly budget. The sweep continues a blocked task behind a refused run. Notices never ask for a Retry the task does not offer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a8b485140
commit
ce4431410b
9 files changed
+235
-11
No files matched your search
@@ -414,4 +414,36 @@ describePostgres("a continuation that fails before doing any work is continued a
|
||||
expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" });
|
||||
expect(await continuationsOf(stopped.id)).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
it("the sweep continues a blocked task whose system retry was refused at claim behind a recovered saved-work hold", async () => {
|
||||
resetBacklogContinuationPacingForTests(db);
|
||||
const f = await seed();
|
||||
const blockedAt = new Date(Date.now() - 20 * MINUTE);
|
||||
const task = await f.task({ status: "blocked", blockedTransitionAt: blockedAt, statusVersion: 3 });
|
||||
const source = await f.run(task.id, { errorCode: "adapter_failed", error: "native_instruction_materialization_handoff_unavailable",
|
||||
finishedAt: new Date(blockedAt.getTime() - 500), resultJson: { workspaceRestoreFailure: "restore_failed" },
|
||||
contextSnapshot: { wakeReason: "issue_children_completed" } });
|
||||
const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog",
|
||||
status: "active", ownerType: "board", returnOwnerAgentId: f.agentId, cause: "legacy_execution_requires_reconciliation",
|
||||
fingerprint: `legacy-execution:${source.id}`, nextAction: "Held.", createdAt: blockedAt,
|
||||
evidence: { runId: source.id, originalFailureCode: "adapter_failed", workspaceRestoreFailure: "restore_failed",
|
||||
systemRecovery: { ...createSystemRecovery(source.id), state: "complete" } } }).returning();
|
||||
const refused = await f.run(task.id, { status: "cancelled", errorCode: "execution_reconciliation_required", startedAt: null,
|
||||
retryOfRunId: source.id, finishedAt: new Date(blockedAt.getTime() + 1_000), createdAt: new Date(blockedAt.getTime() - 100), resultJson: {},
|
||||
contextSnapshot: { wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed" } });
|
||||
const queue = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
|
||||
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(task.id);
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }))
|
||||
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } });
|
||||
const [record] = await continuationsOf(source.id);
|
||||
expect(record?.id).toBe(hold!.id);
|
||||
expect(record?.evidence).toMatchObject({ continuationDelivery: "pending", freshContinuation: { sourceRunId: source.id, trigger: "assigned_work" },
|
||||
freshContinuationSettlement: { refusedAtDispatchRunId: refused.id, requestedBy: "system" } });
|
||||
expect((await taskRow(task.id)).status).toBe("todo");
|
||||
expect(queue.calls).toHaveLength(0);
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }))
|
||||
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent" } });
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -29,6 +29,8 @@ import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapte
|
||||
import { createSystemRecovery } from "./system-recovery-state.js";
|
||||
import { requestTaskRetry } from "./task-retry.js";
|
||||
import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js";
|
||||
import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js";
|
||||
import { taskAutomaticAttempts } from "./recovery/task-automatic-attempts.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
@@ -1323,4 +1325,125 @@ suite("fresh-continuation fallback", () => {
|
||||
const [next] = await successors(f.runId);
|
||||
expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" });
|
||||
}, 60_000);
|
||||
|
||||
async function followUpFailedBeforeProviderWork(options: { restoreFailed?: boolean } = {}) {
|
||||
const issueId = randomUUID(), runId = randomUUID();
|
||||
const endedAt = new Date(Date.now() - 60_000);
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Summarize the subtasks", status: "in_progress", assigneeAgentId: agentId });
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, mode: "isolated",
|
||||
strategyType: "git_worktree", name: "Task workspace", cwd: "/work/remote-task" }).returning();
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issueId));
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "automation", issueId,
|
||||
runtimeMode: "legacy", status: "running", startedAt: new Date(endedAt.getTime() - 10_000),
|
||||
createdAt: new Date(endedAt.getTime() - 60_000), responsibleUserId: userId,
|
||||
contextSnapshot: { issueId, executionWorkspaceId: workspace!.id, wakeReason: "issue_children_completed" } }).returning();
|
||||
const [environment] = await db.insert(environments).values({ name: `sandbox-${runId}`, driver: "sandbox" }).returning();
|
||||
await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace!.id,
|
||||
heartbeatRunId: runId, environmentId: environment!.id, status: "active", leasePolicy: "reuse_by_environment", provider: "daytona",
|
||||
providerLeaseId: `sandbox-${runId}`, metadata: { remoteCwd: "/work/remote-task", driver: "sandbox",
|
||||
workspaceRealization: { authoritativeRoot: "/work/remote-task" } } });
|
||||
await terminalizeLegacyExecution({ db, run: run!, status: "failed", patch: { errorCode: "adapter_failed",
|
||||
error: "native_instruction_materialization_handoff_unavailable", finishedAt: endedAt,
|
||||
resultJson: options.restoreFailed === false ? { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }
|
||||
: { workspaceRestoreFailure: "restore_failed" } } });
|
||||
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
|
||||
return { issueId, runId, actionId: hold?.id ?? null };
|
||||
}
|
||||
|
||||
async function savedWorkRecoveryState(actionId: string, runId: string, state: "running" | "complete") {
|
||||
await db.update(issueRecoveryActions).set({ updatedAt: new Date(),
|
||||
evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ systemRecovery: { ...createSystemRecovery(runId), state } })}::jsonb` })
|
||||
.where(eq(issueRecoveryActions.id, actionId));
|
||||
}
|
||||
|
||||
async function systemRetryAtClaim(f: { issueId: string; runId: string }) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId,
|
||||
runtimeMode: "legacy", status: "queued", retryOfRunId: f.runId,
|
||||
contextSnapshot: { issueId: f.issueId, wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed",
|
||||
retryOfRunId: f.runId } }).returning();
|
||||
const outcome = await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: run!.id,
|
||||
expectedStatus: "queued", now: new Date() });
|
||||
return { run: run!, outcome };
|
||||
}
|
||||
|
||||
async function deliveredContinuationIsClaimable(f: { issueId: string; runId: string; actionId: string | null }, refusedRunId: string) {
|
||||
await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId, issueId: f.issueId });
|
||||
expect((await issue(f.issueId)).status).not.toBe("blocked");
|
||||
expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull();
|
||||
await occupySlot();
|
||||
await deliver();
|
||||
const next = (await successors(f.runId)).filter(run => run.id !== refusedRunId);
|
||||
expect(next).toHaveLength(1);
|
||||
expect(next[0]!.status).toBe("queued");
|
||||
expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: next[0]!.id,
|
||||
expectedStatus: "queued", now: new Date() })).toEqual({ outcome: "not_stale" });
|
||||
return next[0]!;
|
||||
}
|
||||
|
||||
it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered before the system retry was refused at claim", async () => {
|
||||
const f = await followUpFailedBeforeProviderWork();
|
||||
expect(f.actionId).not.toBeNull();
|
||||
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
|
||||
const refused = await systemRetryAtClaim(f);
|
||||
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
|
||||
expect((await holdRow(f.actionId!)).evidence).toMatchObject({
|
||||
freshContinuation: { sourceRunId: f.runId }, continuationDelivery: "pending",
|
||||
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
|
||||
await deliveredContinuationIsClaimable(f, refused.run.id);
|
||||
}, 60_000);
|
||||
|
||||
it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered after the system retry was refused at claim", async () => {
|
||||
const f = await followUpFailedBeforeProviderWork();
|
||||
await savedWorkRecoveryState(f.actionId!, f.runId, "running");
|
||||
const refused = await systemRetryAtClaim(f);
|
||||
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
|
||||
expect((await holdRow(f.actionId!)).evidence.freshContinuationCheck)
|
||||
.toMatchObject({ kind: "exact_recovery_pending", reason: "saved_work_recovery_running" });
|
||||
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
|
||||
await makeDue(f.actionId!);
|
||||
expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 });
|
||||
expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId },
|
||||
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
|
||||
await deliveredContinuationIsClaimable(f, refused.run.id);
|
||||
}, 60_000);
|
||||
|
||||
it("continues a follow-up that failed before provider work with its files in place, after its system retry was refused at claim", async () => {
|
||||
const f = await followUpFailedBeforeProviderWork({ restoreFailed: false });
|
||||
const refused = await systemRetryAtClaim(f);
|
||||
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
|
||||
expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId },
|
||||
freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } });
|
||||
await deliveredContinuationIsClaimable(f, refused.run.id);
|
||||
}, 60_000);
|
||||
|
||||
it("counts only automatic retries that started, or still may, against the task's hourly budget", async () => {
|
||||
const f = await followUpFailedBeforeProviderWork();
|
||||
for (let index = 0; index < FRESH_CONTINUATIONS_PER_HOUR; index++) {
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
|
||||
status: "cancelled", errorCode: "execution_reconciliation_required", finishedAt: new Date(), retryOfRunId: f.runId,
|
||||
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
|
||||
}
|
||||
expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(0);
|
||||
expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(0);
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
|
||||
status: "failed", errorCode: "adapter_failed", startedAt: new Date(), finishedAt: new Date(), retryOfRunId: f.runId,
|
||||
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy",
|
||||
status: "scheduled_retry", retryOfRunId: f.runId,
|
||||
scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } });
|
||||
expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(2);
|
||||
expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(2);
|
||||
}, 60_000);
|
||||
|
||||
it("still leaves a person's failed Retry of a follow-up for that person after its successor is refused at claim", async () => {
|
||||
const f = await followUpFailedBeforeProviderWork();
|
||||
await savedWorkRecoveryState(f.actionId!, f.runId, "complete");
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` })
|
||||
.where(eq(heartbeatRuns.id, f.runId));
|
||||
const refused = await systemRetryAtClaim(f);
|
||||
expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" });
|
||||
expect((await holdRow(f.actionId!)).evidence.freshContinuation).toBeUndefined();
|
||||
expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" }))
|
||||
.toEqual({ kind: "person_owned", reason: "retry_needed" });
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -26,7 +26,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { readRunCancellation } from "./run-cancellation.js";
|
||||
import { readSystemRecovery } from "./system-recovery-state.js";
|
||||
import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuesAutomaticallySql, isPersonOwnedFailureCode } from "./continuation-cause.js";
|
||||
import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js";
|
||||
import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js";
|
||||
|
||||
export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js";
|
||||
@@ -813,7 +813,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ
|
||||
and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at)
|
||||
or exists (select 1 from ${heartbeatRuns} held where ${refusedLater}
|
||||
and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`);
|
||||
const recent = source(sql`${continuesAutomaticallySql("source")}
|
||||
const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")}
|
||||
and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false)
|
||||
and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`;
|
||||
const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource})
|
||||
and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`);
|
||||
return { recent, personAsked };
|
||||
}
|
||||
|
||||
@@ -466,6 +466,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
* Chat conversations and blocks older than the stale window are left out:
|
||||
* the first wait on a person's message, the second is only reported.
|
||||
*/
|
||||
function latestRunRefusedAtDispatch(at: Date) {
|
||||
return sql`coalesce((select lr.status = 'cancelled' and lr.started_at is null and lr.error_code = 'execution_reconciliation_required'
|
||||
and lr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz
|
||||
from heartbeat_runs lr where lr.company_id = ${issues.companyId} and (lr.context_snapshot->>'issueId') = ${issues.id}::text
|
||||
order by lr.created_at desc limit 1), false)`;
|
||||
}
|
||||
|
||||
async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) {
|
||||
const at = now();
|
||||
const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
|
||||
@@ -518,7 +525,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
-- except a recent board escalation, which the predicate judges by its failed run's cause
|
||||
and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr
|
||||
where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId'
|
||||
and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)))`,
|
||||
and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz))
|
||||
and not (${issues.status} = 'blocked' and ra.cause not in ('configuration_incomplete', 'ownership_owner_unprovable')
|
||||
and ${latestRunRefusedAtDispatch(at)}))`,
|
||||
// An assignee that cannot take work (paused, being set up, held) is
|
||||
// its own wait; the predicate names it when anything else asks.
|
||||
sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId}
|
||||
@@ -563,6 +572,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
// hold's owner (the fresh-continuation fallback, same per-task cap). The
|
||||
// sweep reports it and repairs only blocks that have no hold at all, so
|
||||
if (issue.status === "blocked" && facts.activeRecoveryAction && !facts.activeRecoveryAction.continuedBySweep) {
|
||||
const refused = facts.latestRun?.status === "cancelled" && facts.latestRun.errorCode === "execution_reconciliation_required"
|
||||
? await continueFromSavedWork(issue, facts, at) : null;
|
||||
if (refused?.kind === "continued" || refused?.kind === "already_continued") return { verdict, outcome: "repaired", retryAt: null };
|
||||
if (refused?.kind === "backoff" || refused?.kind === "exact_recovery_pending")
|
||||
return { verdict, outcome: refused.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: refused.retryAt };
|
||||
return { verdict, outcome: "left_to_hold_owner", retryAt: null };
|
||||
}
|
||||
// A stalled native run is ended by native recovery, not here: the sweep
|
||||
@@ -624,8 +638,21 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
|
||||
async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise<FreshContinuationResult | null> {
|
||||
const source = facts.latestRun;
|
||||
if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)
|
||||
|| (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required")) return null;
|
||||
if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)) return null;
|
||||
if (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required") {
|
||||
const [refusedFor] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, issue.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`,
|
||||
inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), sql`${heartbeatRuns.finishedAt} is not null`,
|
||||
sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null
|
||||
and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`))
|
||||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
|
||||
if (!refusedFor) return null;
|
||||
return ensureFreshContinuation(db, { companyId: issue.companyId, runId: refusedFor.id, trigger: "assigned_work", now: at })
|
||||
.catch(error => {
|
||||
logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation behind a refused run deferred");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error,
|
||||
retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns)
|
||||
|
||||
@@ -6,6 +6,11 @@ export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure";
|
||||
export const AUTOMATIC_RETRY_REASONS = [SCHEDULER_TRANSIENT_RETRY_REASON, "server_restart", "interrupted_preparation"];
|
||||
export const SHARED_CONTINUATION_WINDOW_MS = 60 * 60_000;
|
||||
|
||||
export function startedOrPendingAutomaticRetry() {
|
||||
return sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null
|
||||
and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`;
|
||||
}
|
||||
|
||||
export async function freshContinuationTimes(db: Db, companyId: string, issueId: string, since: Date) {
|
||||
const rows = await db.select({ recordedAt: sql<string>`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`,
|
||||
trigger: sql<string | null>`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}'` }).from(issueRecoveryActions).where(and(
|
||||
@@ -25,7 +30,8 @@ export async function sharedContinuationAttempts(db: Db, companyId: string, issu
|
||||
sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`)).limit(64),
|
||||
db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))).limit(64),
|
||||
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since),
|
||||
startedOrPendingAutomaticRetry())).limit(64),
|
||||
freshContinuationTimes(db, companyId, issueId, since),
|
||||
]);
|
||||
return [...plainContinuations.map(row => row.createdAt), ...retries.map(row => row.createdAt), ...fresh.map(record => record.at)]
|
||||
|
||||
@@ -176,6 +176,7 @@ import {
|
||||
buildExecutionReviewParticipantRecoveryNoticeSeed,
|
||||
buildExecutionReviewParticipantUnavailableNoticeSeed,
|
||||
buildStrandedRecoveryEscalationNotice,
|
||||
withoutRetryInstruction,
|
||||
buildNoRunningExecutionNoticeSeed,
|
||||
RETRY_FROM_SAVED_WORK_NEXT_ACTION,
|
||||
type StrandedRecoveryNoticeSeed,
|
||||
@@ -185,6 +186,7 @@ import {
|
||||
isStrandedIssueRecoveryOriginKind,
|
||||
} from "./origins.js";
|
||||
import { withRecoveryContext } from "./status-only-context.js";
|
||||
import { getExecutionBlocker } from "../execution-blocker.js";
|
||||
import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js";
|
||||
import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js";
|
||||
import { rehomeStrandedLegacyWorkspaceTasks } from "./legacy-workspace-rehome.js";
|
||||
@@ -4250,6 +4252,13 @@ export function recoveryService(
|
||||
recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
}, tx);
|
||||
const executionBlocker = await getExecutionBlocker(tx as unknown as Db, input.issue.companyId, input.issue.id);
|
||||
const retryOffered = !(executionBlocker && executionBlocker.canRetry === false);
|
||||
if (!retryOffered && /\bretry\b/i.test(recoveryAction.nextAction ?? "")) {
|
||||
recoveryAction.nextAction = withoutRetryInstruction(recoveryAction.nextAction!);
|
||||
await tx.update(issueRecoveryActions).set({ nextAction: recoveryAction.nextAction })
|
||||
.where(and(eq(issueRecoveryActions.companyId, input.issue.companyId), eq(issueRecoveryActions.id, recoveryAction.id)));
|
||||
}
|
||||
const isProviderQuotaWait =
|
||||
recoveryCause === "provider_quota" &&
|
||||
!recoveryAction.ownerAgentId &&
|
||||
@@ -4333,6 +4342,7 @@ export function recoveryService(
|
||||
fallbackBody: input.comment,
|
||||
recoveryCause,
|
||||
recoveryActionId: recoveryAction.id,
|
||||
retryOffered,
|
||||
continuesAutomatically: Boolean(input.latestRun) && recoveryAction.kind === STRANDED_KIND
|
||||
&& recoveryAction.ownerType === "board" && input.latestRun?.agentId === input.issue.assigneeAgentId
|
||||
&& strandedRunCauseIsTemporary({
|
||||
|
||||
@@ -401,3 +401,16 @@ it("tells the operator to switch a preconfigured Runner home to an isolated one"
|
||||
expect(notice.nextAction).toContain("isolated Runner home");
|
||||
expect(notice.nextAction).not.toContain("Choose an AI connection");
|
||||
});
|
||||
|
||||
it("never tells a person to press Retry when the task offers no Retry", () => {
|
||||
const sourceRun = { id: "run-1", status: "cancelled", errorCode: "execution_reconciliation_required" };
|
||||
for (const seed of [buildNoRunningExecutionNoticeSeed({ status: "in_progress" }), null]) {
|
||||
const offered = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun });
|
||||
expect(`${offered.body} ${JSON.stringify(allRows(offered.metadata))}`).toMatch(/Press Retry/);
|
||||
const withheld = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun,
|
||||
retryOffered: false });
|
||||
expect(withheld.body).not.toMatch(/retry/i);
|
||||
expect(JSON.stringify(allRows(withheld.metadata))).not.toMatch(/retry/i);
|
||||
expect(withheld.body).toContain("Paperclip is holding this task until its saved work is settled.");
|
||||
}
|
||||
});
|
||||
@@ -27,6 +27,13 @@ export type StrandedRecoveryNoticeSeed = {
|
||||
nextAction?: string;
|
||||
};
|
||||
|
||||
export const SAVED_WORK_HELD_SENTENCE = "Paperclip is holding this task until its saved work is settled.";
|
||||
|
||||
export function withoutRetryInstruction(text: string): string {
|
||||
const kept = text.split(/(?<=[.!?])\s+/).filter(sentence => !/\bretry\b/i.test(sentence));
|
||||
return [...kept, SAVED_WORK_HELD_SENTENCE].join(" ");
|
||||
}
|
||||
|
||||
export type StrandedRecoveryEscalationNotice = {
|
||||
body: string;
|
||||
presentation: IssueCommentPresentation;
|
||||
@@ -270,6 +277,7 @@ export function buildStrandedRecoveryEscalationNotice(input: {
|
||||
recoveryCause?: string | null;
|
||||
recoveryActionId: string;
|
||||
continuesAutomatically?: boolean;
|
||||
retryOffered?: boolean;
|
||||
recoveryOwner: { id: string; name: string | null } | null | undefined;
|
||||
sourceRun: {
|
||||
id: string;
|
||||
@@ -292,7 +300,8 @@ export function buildStrandedRecoveryEscalationNotice(input: {
|
||||
tone: "danger" as const,
|
||||
} : SETUP_FAILURE_NOTICES[input.sourceRun?.errorCode ?? ""] ?? input.seed;
|
||||
const fallbackBody = input.fallbackBody?.trim();
|
||||
const body = seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY);
|
||||
const honest = (text: string) => input.retryOffered === false ? withoutRetryInstruction(text) : text;
|
||||
const body = honest(seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY));
|
||||
const title =
|
||||
STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE[input.sourceRun?.errorCode?.trim() ?? ""] ??
|
||||
seed?.title ??
|
||||
@@ -310,9 +319,9 @@ export function buildStrandedRecoveryEscalationNotice(input: {
|
||||
),
|
||||
keyValueRow(
|
||||
"Next action",
|
||||
seed?.nextAction ?? (input.recoveryOwner
|
||||
honest(seed?.nextAction ?? (input.recoveryOwner
|
||||
? "The assigned agent continues the task from the saved work"
|
||||
: RETRY_FROM_SAVED_WORK_NEXT_ACTION),
|
||||
: RETRY_FROM_SAVED_WORK_NEXT_ACTION)),
|
||||
),
|
||||
];
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, desc, eq, gte, inArray, sql } from "drizzle-orm";
|
||||
import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes } from "./continuation-ledger.js";
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes, startedOrPendingAutomaticRetry } from "./continuation-ledger.js";
|
||||
import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js";
|
||||
|
||||
export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON };
|
||||
@@ -14,7 +14,8 @@ export async function taskAutomaticAttempts(db: Db, companyId: string, issueId:
|
||||
.orderBy(desc(activityLog.createdAt)).limit(64),
|
||||
db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since)))
|
||||
inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since),
|
||||
startedOrPendingAutomaticRetry()))
|
||||
.orderBy(desc(heartbeatRuns.createdAt)).limit(64),
|
||||
]);
|
||||
const fresh = (await freshContinuationTimes(db, companyId, issueId, since))
|
||||
|
||||
Reference in new issue
Block a user