diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index a94b211d0f6..52d1d9b1367 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -472,8 +472,9 @@ describePostgres("assigned-work invariant sweep", () => { evidence: { runId: run.id, originalFailureCode: "process_lost" }, nextAction: "Reconcile the stopped run.", attemptCount: 1 }); const heartbeat = heartbeatService(db); try { + // The wake became the continuation, so the sweep counts it as repaired. const first = await heartbeat.checkAssignedWorkInvariant({ companyId: f.companyId, issueId }); - expect(first).toMatchObject({ outcome: "enqueue_refused", verdict: { kind: "violation" } }); + expect(first).toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } }); const queued = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sqlTag`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`)); expect(queued).toHaveLength(0); @@ -482,7 +483,7 @@ describePostgres("assigned-work invariant sweep", () => { expect(pending).toHaveLength(0); const [entry] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); - expect(entry.details).toMatchObject({ outcome: "not_started" }); + expect(entry.details).toMatchObject({ outcome: "queued", continuedFromSavedWork: "true" }); const [record] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId), sqlTag`${issueRecoveryActions.evidence}->'freshContinuation' is not null`)); expect(record!.evidence).toMatchObject({ continuationDelivery: "pending" }); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 12b197ed3d4..87b58d0f21e 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -25,8 +25,9 @@ import { abandonUnverifiedRemoteWorkspaceEpoch } from "./remote-workspace-epoch- import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; -import { readSystemRecovery } from "./system-recovery-state.js"; -import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; +import { PERMANENT_SOURCE_RETRY_CLASSES, UNVERIFIABLE_SOURCE_CODES, readSystemRecovery } from "./system-recovery-state.js"; +import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, + isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; @@ -806,6 +807,8 @@ const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${ali const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id = ${issueId}::uuid or ${sql.raw(alias)}.issue_id = ${issueId}::uuid or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; +const sqlList = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); + function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} @@ -820,7 +823,19 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; - const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) + // Saved-work recovery proved the run's unsaved files are permanently gone + // (`source_unavailable`, use another strategy). The only way on is a fresh + // run in a new workspace, which is exactly what Retry would do: a person + // adds nothing, so the system takes that step itself (needed-retry is 0%). + // Uncast keys, so the (company, task, evidence runId) index serves the lookup. + const savedFilesLost = sql`exists (select 1 from ${issueRecoveryActions} lost where lost.company_id = source.company_id + and lost.source_issue_id = ${input.issueId} + and (lost.evidence->>'runId' = source.id::text or (lost.evidence->>'runId' is null and lost.evidence->>'sourceRunId' = source.id::text)) + and lost.evidence#>>'{systemRecovery,state}' = 'exhausted' + and lost.evidence#>>'{systemRecovery,failure,code}' in (${sqlList(UNVERIFIABLE_SOURCE_CODES)}) + and lost.evidence#>>'{systemRecovery,failure,retryClass}' in (${sqlList(PERMANENT_SOURCE_RETRY_CLASSES)}))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource} + or (${continuationCauseIsSystemOwnedSql("source")} and ${savedFilesLost})) and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); return { recent, personAsked }; } diff --git a/server/src/services/fresh-continuation-schedule-queries.test.ts b/server/src/services/fresh-continuation-schedule-queries.test.ts index 0b894b8c060..7171025de75 100644 --- a/server/src/services/fresh-continuation-schedule-queries.test.ts +++ b/server/src/services/fresh-continuation-schedule-queries.test.ts @@ -82,11 +82,12 @@ function random(seed: number) { type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number; "Rows Removed by Filter"?: number; "Rows Removed by Index Recheck"?: number; Plans?: PlanNode[] }; -function heartbeatRunRowsRead(node: PlanNode): number { - const own = node["Relation Name"] === "heartbeat_runs" ? (node["Actual Loops"] ?? 0) +function rowsRead(node: PlanNode, relation: string): number { + const own = node["Relation Name"] === relation ? (node["Actual Loops"] ?? 0) * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0) + (node["Rows Removed by Index Recheck"] ?? 0)) : 0; - return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0); + return own + (node.Plans ?? []).reduce((total, child) => total + rowsRead(child, relation), 0); } +const heartbeatRunRowsRead = (node: PlanNode) => rowsRead(node, "heartbeat_runs"); suite("fresh-continuation schedule queries", () => { let db!: ReturnType; @@ -244,7 +245,7 @@ suite("fresh-continuation schedule queries", () => { expect(await heldExecutionsAwaitingFreshContinuation(db, now, 1_000)).toEqual(previousListing); }, 60_000); - it("finds the deadline by run-key and task-link lookups, not a scan of run history per record", async () => { + async function deadlinePlan() { let captured: { sql: string; params: unknown[] } | undefined; const capture = new Proxy(db, { get(target, key) { if (key !== "select") return Reflect.get(target, key); @@ -260,10 +261,46 @@ suite("fresh-continuation schedule queries", () => { } }); await nextFreshContinuationCheckAt(capture as typeof db, now); const [row] = await db.$client.unsafe(`explain (analyze, format json) ${captured!.sql}`, captured!.params as never[]); - const plan = (row!["QUERY PLAN"] as Array<{ Plan: PlanNode }>)[0]!.Plan; + return (row!["QUERY PLAN"] as Array<{ Plan: PlanNode }>)[0]!.Plan; + } + + it("finds the deadline by run-key and task-link lookups, not a scan of run history per record", async () => { + const plan = await deadlinePlan(); const [{ runs }] = await db.$client.unsafe(`select count(*)::int as runs from heartbeat_runs`) as unknown as Array<{ runs: number }>; // The previous query read the whole run history once per held record // (hundreds of times here); the lookups read a few rows per record. expect(heartbeatRunRowsRead(plan)).toBeLessThan(2 * runs); }, 60_000); + + // Runs last: its candidates reach the saved-files-lost check, which the + // previous predicates above do not have. + it("checks for permanently lost saved files by the task's own records, not every recovery record per candidate", async () => { + const { companyId, agentIds } = tenants[0]!; + const fillerIssues = Array.from({ length: 100 }, (_, index) => ({ id: randomUUID(), companyId, title: `Filler ${index}`, status: "done" })); + await db.insert(issues).values(fillerIssues as never); + const filler = Array.from({ length: 3_000 }, (_, index) => ({ companyId, sourceIssueId: fillerIssues[index % 100]!.id, kind: "missing_disposition", + status: "resolved", ownerType: "board", cause: "unrelated_fixture", fingerprint: `filler:${index}`, nextAction: "Fixture", + evidence: { runId: randomUUID() } })); + for (let start = 0; start < filler.length; start += 500) + await db.insert(issueRecoveryActions).values(filler.slice(start, start + 500) as never); + for (let index = 0; index < 30; index += 1) { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: `Lost files ${index}`, assigneeAgentId: agentIds[0]!, status: "in_progress" }); + // A person's Retry that failed in setup: system-owned, but not an automatic continuation. + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: agentIds[0]!, scopeKind: "issue", issueId, status: "failed", + errorCode: "setup_failed", startedAt: new Date(now.getTime() - 40 * 60_000), finishedAt: new Date(now.getTime() - 30 * 60_000), + createdAt: new Date(now.getTime() - 41 * 60_000), contextSnapshot: { issueId, wakeReason: "task_retry" } }); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", status: "resolved", + ownerType: "board", cause: "legacy_execution_requires_reconciliation", fingerprint: `lost:${index}`, nextAction: "Fixture", + evidence: { runId, automaticRecovery: { replay: "blocked" }, + systemRecovery: { state: "exhausted", failure: { code: "source_unavailable", retryClass: "use_alternative_strategy" } }, + freshContinuationCheck: { kind: "backoff", nextCheckAt: now.getTime() - 60_000 } } }); + } + await db.execute(sql`analyze`); + const plan = await deadlinePlan(); + const [{ holds }] = await db.$client.unsafe(`select count(*)::int as holds from issue_recovery_actions`) as unknown as Array<{ holds: number }>; + // A cast on the indexed keys read every recovery record once per candidate (about 30 × 4,000 here). + expect(rowsRead(plan, "issue_recovery_actions")).toBeLessThan(2 * holds); + expect(await nextFreshContinuationCheckAt(db, now)).toBeLessThanOrEqual(now.getTime() - 60_000); + }, 120_000); }); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 0f9fe34c63d..8026aed31fb 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -293,6 +293,17 @@ export async function terminalizeLegacyExecution(input: { )).orderBy(desc(issueRecoveryActions.createdAt)).limit(1).for("update"); const decision = existing?.evidence.executionReconciliation as Record | undefined; if (hasRequiredWorkspaceRecovery(existing?.evidence) && decision?.runId === run.id) return updated; + // The stranded sweep revisits this terminal run every pass. The hold + // already records this exact no-replay decision, or a fresh continuation + // already took it over: rewriting it would only churn its timestamps and + // reset a coalesced continuation's delivery state. + const recorded = existing?.evidence.automaticRecovery as Record | undefined; + const continuation = existing?.evidence.freshContinuation as Record | undefined; + if (existing && (continuation?.sourceRunId === run.id || (existing.status === "resolved" + && recorded?.runId === run.id && recorded.replay === "blocked" + && existing.evidence.workspaceRestoreFailure === updated.resultJson!.workspaceRestoreFailure + && JSON.stringify(existing.evidence.workspaceRestoreRecovery) === JSON.stringify(updated.resultJson!.workspaceRestoreRecovery)))) + return updated; const now = new Date(); const evidence = { ...existing?.evidence, diff --git a/server/src/services/legacy-source-unavailable-continuation.test.ts b/server/src/services/legacy-source-unavailable-continuation.test.ts new file mode 100644 index 00000000000..f33bc7006db --- /dev/null +++ b/server/src/services/legacy-source-unavailable-continuation.test.ts @@ -0,0 +1,146 @@ +import { and, eq, sql } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, environmentLeases, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { recoverRetainedWorkspace } from "./workspace-recovery.js"; +import { heartbeatService } from "./heartbeat.js"; +import { deliverReconciledExecutions } from "./execution-recovery-resolution.js"; +import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; +import { assignedWorkSweep, ASSIGNED_WORK_CONTINUED_ACTION } from "./recovery/assigned-work-sweep.js"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { legacyReconciliationRunFields, terminalizeLegacyExecution } from "./legacy-execution-recovery.js"; +import { action, connectionString, db, failedRemotePreparation, run, useWorkspaceRecoveryFixture, withheldPreparationRetry } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +/** + * Staging, 2026-11-13: a legacy run on Daytona failed in setup and + * its files were held for saved-work recovery. The sandbox was gone, so the + * recovery ended `source_unavailable` (use another strategy). The task then + * offered Retry, and Retry worked: it continued in a fresh workspace. But + * nothing continued it automatically. For 2 hours the assigned-work sweep + * logged `retry_offered` / `not_started` every 15 minutes, because the hold + * refused its wake. A person had to press Retry. + */ +describe("a legacy run whose saved files are gone", () => { + useWorkspaceRecoveryFixture(targets); + beforeEach(() => { + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); + }); + afterEach(() => { vi.unstubAllEnvs(); }); + + async function sourceGone(withheldRetry = true) { + const f = await failedRemotePreparation(); + // The staging run was itself a person's Retry, so it is not an automatic + // recovery run, and the failure (setup_failed) is not a restart. + const source = await run(f.runId); + const finishedAt = new Date(Date.now() - 45 * 60_000); + await db.update(heartbeatRuns).set({ finishedAt, startedAt: new Date(finishedAt.getTime() - 10 * 60_000), + createdAt: new Date(finishedAt.getTime() - 11 * 60_000), + contextSnapshot: { ...source.contextSnapshot, wakeReason: "task_retry" } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ status: "in_progress", statusVersion: 3 }).where(eq(issues.id, f.issueId)); + // The provider destroyed the sandbox: its files can never be read again. + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, + remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: f.providerLeaseId, state: "destroyed" }) } }) + .where(eq(environmentLeases.id, f.lease.id)); + await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", + failure: { code: "source_unavailable", retryClass: "use_alternative_strategy" } }); + // The hold withheld the scheduled retry that came after it. + if (!withheldRetry) return f; + const withheld = await withheldPreparationRetry(f); + await db.update(heartbeatRuns).set({ createdAt: new Date(finishedAt.getTime() + 60_000), + finishedAt: new Date(finishedAt.getTime() + 2 * 60_000) }).where(eq(heartbeatRuns.id, withheld.id)); + return f; + } + + /** What the stranded-work sweep does on every pass for this failed run. */ + async function strandedPass(runId: string) { + const [source] = await db.select(legacyReconciliationRunFields).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await terminalizeLegacyExecution({ db, run: source, status: source.status, fromStatuses: [source.status] }); + } + + it("continues the task in a fresh workspace without a person's Retry", async () => { + const f = await sourceGone(); + const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]!; + const heartbeat = heartbeatService(db); + const sweep = assignedWorkSweep(db, { enqueueWakeup: (agentId, opts) => heartbeat.wakeup(agentId, opts) }); + const check = await sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId }); + // The sweep's wake becomes the fresh continuation (the same path Retry + // takes), instead of a wake the hold refuses. + expect(check).toMatchObject({ verdict: { reason: "retry_offered" }, outcome: "repaired" }); + // The stranded sweep revisiting the failed run keeps the recorded continuation. + await strandedPass(f.runId); + expect((await action(f.actionId)).evidence).toMatchObject({ continuationDelivery: "pending", + freshContinuation: { sourceRunId: f.runId }, executionReconciliation: { runId: f.runId } }); + // The continuation worker starts the recorded continuation. + await deliverReconciledExecutions(db, (agentId, opts) => heartbeat.wakeup(agentId, opts)); + // A run continues the task, in a workspace that is not the lost sandbox. + await vi.waitFor(async () => { + const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`, + sql`${heartbeatRuns.errorCode} is distinct from 'execution_reconciliation_required'`)); + expect(successor).toBeDefined(); + }, { timeout: 10_000, interval: 200 }); + // The task no longer asks a person to press Retry. + expect((await taskInterruptionPresentation(db, issue))?.canRetry ?? false).toBe(false); + const refused = await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), sql`${activityLog.details}->>'outcome' = 'not_started'`)); + expect(refused).toHaveLength(0); + expect((await run(f.runId)).status).toBe("failed"); + }, 90_000); + + it("leaves the hold as it is when the stranded sweep revisits the failed run", async () => { + const f = await sourceGone(); + const before = await action(f.actionId); + await strandedPass(f.runId); + await strandedPass(f.runId); + const after = await action(f.actionId); + expect(after.evidence).toEqual(before.evidence); + expect(after.updatedAt).toEqual(before.updatedAt); + // A changed decision is still recorded: a hold that lost it gets it back. + await db.update(issueRecoveryActions).set({ evidence: { ...before.evidence, automaticRecovery: undefined } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await strandedPass(f.runId); + expect((await action(f.actionId)).evidence).toMatchObject({ automaticRecovery: { runId: f.runId, replay: "blocked" } }); + }, 90_000); + + it("counts the continuation as started when its worker delivers it before the sweep looks", async () => { + // No withheld retry: the sweep's own wake reaches the fallback in admission. + const f = await sourceGone(false); + const heartbeat = heartbeatService(db); + // The continuation worker delivers the run between admission and the sweep's read. + const sweep = assignedWorkSweep(db, { enqueueWakeup: async (agentId, opts) => { + const run = await heartbeat.wakeup(agentId, opts); + await deliverReconciledExecutions(db, (id, wake) => heartbeat.wakeup(id, wake)); + return run; + } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })).toMatchObject({ outcome: "repaired" }); + expect((await action(f.actionId)).evidence).toMatchObject({ continuationDelivery: "delivered" }); + const refused = await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), sql`${activityLog.details}->>'outcome' = 'not_started'`)); + expect(refused).toHaveLength(0); + }, 90_000); + + it("does not report a continuation again once it is stuck waiting for delivery", async () => { + const f = await sourceGone(false); + const heartbeat = heartbeatService(db); + let clock = Date.now(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: (agentId, opts) => heartbeat.wakeup(agentId, opts), now: () => new Date(clock) }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })).toMatchObject({ outcome: "repaired" }); + // Nothing delivers it. Once its pending bound passes, a new wake adds nothing. + clock += 2 * 60 * 60_000; + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })).not.toMatchObject({ outcome: "repaired" }); + const queued = await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), sql`${activityLog.details}->>'outcome' = 'queued'`)); + expect(queued).toHaveLength(1); + }, 90_000); +}); diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index c3ca5f80ba1..3315703383e 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -39,6 +39,7 @@ import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } import { ASSIGNED_WORK_CONTINUED_ACTION, PAUSED_FOR_AUTOMATIC_WORK, taskAutomaticAttempts } from "./task-automatic-attempts.js"; import { automaticRunCreationHeldUntil, readAutomaticWorkStatus, reserveAutomaticRunCreation } from "./automatic-run-brake.js"; import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; +import { wakeBecameFreshContinuation } from "./wake-became-fresh-continuation.js"; import { continueSweepTaskFromSavedWork } from "./sweep-saved-work-continuation.js"; import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { @@ -690,6 +691,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil }; } const release = async () => { if (slot && !slot.heldUntil) await slot.release(); }; + const continuedBefore = await wakeBecameFreshContinuation(db, issue.companyId, issue.id); const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system", reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", @@ -702,14 +704,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation was not admitted"); return null; }); - // A refused continuation stays a violation; the claim spaces the next try. - // A block lifted for it is put back, so the task never sits unblocked - // with nothing pending. + // Admission may coalesce this wake into a new fresh continuation from saved work (not a refusal). An older + // stuck one, or a refused wake, stays a violation spaced by the claim; a block lifted for it is put back. + const continued = !run && !continuedBefore && await wakeBecameFreshContinuation(db, issue.companyId, issue.id); if (!run) await release(); - if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); - await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", - { ...(fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}), ...(run ? {} : refusal) }); - return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; + if (!run && !continued && claimed.restoredTo) await revertRestoredBlock(issue, claimed); + await settleContinuationActivity(claimed, claimed.reason, run || continued ? "queued" : "not_started", continued + ? { continuedFromSavedWork: "true" } + : { ...(fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}), ...(run ? {} : refusal) }); + return { verdict: claimed.verdict, outcome: run || continued ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } function stateKey(facts: AssignedWorkFacts, verdict: Extract) { diff --git a/server/src/services/recovery/wake-became-fresh-continuation.ts b/server/src/services/recovery/wake-became-fresh-continuation.ts new file mode 100644 index 00000000000..723561b205b --- /dev/null +++ b/server/src/services/recovery/wake-became-fresh-continuation.ts @@ -0,0 +1,29 @@ +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { canonicalExecutionUuid } from "../execution-identity.js"; + +/** + * True when admission coalesced a continuation wake into a fresh continuation + * from saved work. The continuation is still pending (admission already woke + * its worker), or the worker delivered it as a run before this read. The + * caller proved no run of the task was live before its wake, so a live + * delivered continuation run is that wake's work. One read, no transaction: + * the sweep runs this after every refused wake. + */ +export async function wakeBecameFreshContinuation(db: Db, companyId: string, issueId: string): Promise { + const delivery = sql`${issueRecoveryActions.evidence}->>'continuationDelivery'`; + const [row] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' is not null`, + sql`((${issueRecoveryActions.status} = 'resolved' and ${delivery} = 'pending') + or (${delivery} = 'delivered' and exists (select 1 from ${heartbeatRuns} continuation + where continuation.company_id = ${issueRecoveryActions.companyId} + and continuation.id = ${canonicalExecutionUuid(sql`${issueRecoveryActions.evidence}->>'continuationRunId'`)} + and continuation.status in ('queued', 'running'))))`)).limit(1) + .catch(error => { + logger.warn({ err: error, companyId, issueId }, "assigned-work sweep could not read the task's continuation"); + return []; + }); + return Boolean(row); +}