diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 4261c4b1598..a94b211d0f6 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -446,7 +446,7 @@ describePostgres("assigned-work invariant sweep", () => { expect(deadlines).toHaveLength(1); // Too soon: spaced, not repaired and not reported as fine. clock = new Date(clock.getTime() + 60_000); - expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "deferred_rate_limit", verdict: { kind: "violation" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "waiting_unchanged", verdict: { kind: "violation" } }); // After the spacing it is admitted; the queue later declines the claim // (for example behind the old run's pending cleanup) and the wake is skipped. admit = true; diff --git a/server/src/__tests__/assigned-work-recheck-idle.test.ts b/server/src/__tests__/assigned-work-recheck-idle.test.ts new file mode 100644 index 00000000000..90ccd0b4314 --- /dev/null +++ b/server/src/__tests__/assigned-work-recheck-idle.test.ts @@ -0,0 +1,109 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, desc, eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { activityLog, agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION } from "../services/recovery/assigned-work-sweep.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +describeIssueRecovery("assigned-work re-checks of tasks whose continuation is refused", () => { + const state = issueRecoveryFixture(); + const TASKS = 10; + + async function seed() { + const f = await state.fixture(); + const db = state.db; + await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } } }) + .where(eq(agents.id, f.agentId)); + const tasks = await db.insert(issues).values(Array.from({ length: TASKS }, (_, index) => ({ + companyId: f.companyId, title: `Refused task ${index}`, status: "in_progress", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + await db.insert(heartbeatRuns).values(tasks.map(task => ({ companyId: f.companyId, agentId: f.agentId, + status: "failed", errorCode: "adapter_failed", finishedAt: new Date(Date.now() - HOUR), contextSnapshot: { issueId: task.id } }))); + return { ...f, tasks }; + } + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(500, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + async function countStatements(work: () => Promise) { + const statements: string[] = [], prior = state.db.$client.options.debug; + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { await work(); } finally { state.db.$client.options.debug = prior; } + return statements; + } + + const attempts = async (companyId: string, issueId?: string) => (await state.db.select().from(activityLog).where(and( + eq(activityLog.companyId, companyId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), + issueId ? eq(activityLog.entityId, issueId) : undefined))).length; + + async function boot(f: Awaited>) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(5 * MINUTE); + expect(await attempts(f.companyId)).toBeGreaterThanOrEqual(TASKS); + const report = await state.heartbeat.reportAssignedWorkInvariant(f.companyId); + expect(report.violations).toEqual({ retry_offered: TASKS }); + } + + it("backs off while every check finds the same refused state", async () => { + const f = await seed(); + await boot(f); + await advance(HOUR); + const before = await attempts(f.companyId); + const idle = await countStatements(() => advance(5 * HOUR)); + expect(await attempts(f.companyId) - before).toBeLessThanOrEqual(3 * TASKS); + expect(idle.length).toBeLessThan(TASKS * 500); + const [latest] = await state.db.select().from(activityLog).where(and(eq(activityLog.entityId, f.tasks[0]!.id), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))).orderBy(desc(activityLog.createdAt)).limit(1); + expect(latest?.details).toMatchObject({ outcome: "not_started", stateKey: expect.any(String), + message: expect.stringContaining("still could not start") }); + }, 600_000); + + it("re-checks a backed-off task within two minutes when one of its runs finishes", async () => { + const f = await seed(); + await boot(f); + await advance(3 * HOUR); + const target = f.tasks[0]!.id; + const before = await attempts(f.companyId, target); + await state.db.transaction(async tx => { + const [run] = await tx.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "failed", + errorCode: "adapter_failed", finishedAt: new Date(), contextSnapshot: { issueId: target } }).returning(); + await notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id }); + }); + await advance(2 * MINUTE); + expect(await attempts(f.companyId, target)).toBeGreaterThan(before); + }, 600_000); + + it("leaves review work alone and re-checks a refused task with reads only while it waits", async () => { + const f = await seed(); + const db = state.db; + const [paused] = await db.insert(agents).values({ companyId: f.companyId, name: "Paused reviewer", role: "engineer", status: "paused", + adapterType: "process", adapterConfig: {} }).returning(); + await db.insert(issues).values(Array.from({ length: 60 }, (_, index) => ({ companyId: f.companyId, title: `Review ${index}`, + status: "in_review", assigneeAgentId: f.agentId, executionState: { status: "pending" } }))); + const running = await db.insert(issues).values(Array.from({ length: 10 }, (_, index) => ({ companyId: f.companyId, + title: `Running ${index}`, status: "in_progress", assigneeAgentId: f.agentId }))).returning({ id: issues.id }); + await db.insert(heartbeatRuns).values(running.map(task => ({ companyId: f.companyId, agentId: f.agentId, status: "running", + startedAt: new Date(), contextSnapshot: { issueId: task.id } }))); + await db.insert(issues).values(Array.from({ length: 5 }, (_, index) => ({ companyId: f.companyId, title: `Paused ${index}`, + status: "todo", assigneeAgentId: paused!.id, createdAt: new Date(Date.now() - HOUR) }))); + const report = await state.heartbeat.reportAssignedWorkInvariant(f.companyId); + expect(report).toMatchObject({ openTasks: 25, violations: { retry_offered: TASKS }, + ok: { live_run: 10, "waiting:agent_paused": 5 } }); + const target = f.tasks[0]!.id; + const claim = await countStatements(() => state.heartbeat.checkAssignedWorkInvariant({ companyId: f.companyId, issueId: target })); + const recheck = await countStatements(() => state.heartbeat.checkAssignedWorkInvariant({ companyId: f.companyId, issueId: target })); + expect(claim.some(query => /^insert into "activity_log"/i.test(query))).toBe(true); + expect(recheck.filter(query => /^(insert|update|delete)/i.test(query.trim()))).toEqual([]); + expect(recheck.length).toBeLessThan(claim.length); + }, 600_000); +}); diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 317f69c5516..9bca61b2f82 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -177,7 +177,7 @@ export type AssignedWorkFacts = { /** A saved-work recovery the recovery queue still owes this task: the * task's saved files must be recovered before any run can start. */ savedWorkRecovery?: SavedWorkRecoveryFact | null; - agent: { status: string; invokable: boolean } | null; + agent: { status: string; invokable: boolean; updatedAt?: Date } | null; companyPaused: boolean; pauseHold: boolean; budgetStop: boolean; @@ -385,6 +385,8 @@ export const CONTINUATION_REVIEW_MESSAGE = "Returned to review automatically after an interruption. Nothing needed you, so the review picks up where it stopped."; export const CONTINUATION_REFUSED_MESSAGE = "Paperclip tried to continue this task automatically, but it could not start yet. It will try again shortly."; +export const CONTINUATION_REFUSED_AGAIN_MESSAGE = + "Paperclip still could not start this task. It will keep trying, less often each time, and right away when something on the task changes."; /** Plain-language activity text for an admitted continuation. No codes, no ids. */ export function continuationActivityMessage(reason: ViolationReason): string { diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 9317316ab80..c3ca5f80ba1 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { and, desc, eq, gte, inArray, isNull, or, sql, type SQL, type SQLWrapper } from "drizzle-orm"; import { activityLog, @@ -47,6 +48,8 @@ import { SAVED_WORK_RECOVERY_START_BOUND_MS, savedWorkRecoveryDueSince, CONTINUATION_REFUSED_MESSAGE, + CONTINUATION_REFUSED_AGAIN_MESSAGE, + CONTINUATION_BASE_BACKOFF_MS, CONTINUATION_REVIEW_MESSAGE, CONTINUATION_CAP_WINDOW_MS, evaluateAssignedWork, @@ -69,6 +72,8 @@ export const STRANDED_TRANSIENT_CONTINUED_OUTCOME = "stranded_transient_continue export const ASSIGNED_WORK_REPAIRS_PER_MINUTE = 10; /** Upper bound on tasks one report reads. */ export const ASSIGNED_WORK_REPORT_LIMIT = 2_000; +export const UNCHANGED_REFUSAL_MAX_WAIT_MS = 6 * 60 * 60_000; +const UNCHANGED_REFUSAL_CHAIN = 9; const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const; const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"]; @@ -108,7 +113,7 @@ export type AssignedWorkCheck = { verdict: AssignedWorkVerdict; outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "deferred_reattach_phase" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" | "waiting_on_recovery" | "started_saved_work_recovery" | "saved_work_recovery_busy" - | "replaced_stalled_run" | "left_person_work" | "left_stalled_replacement"; + | "replaced_stalled_run" | "left_person_work" | "left_stalled_replacement" | "waiting_unchanged"; retryAt: Date | null; }; @@ -458,7 +463,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { resumeAt: PROVIDER_LIMIT_CAUSES.includes(action.cause) && interruption?.nextRetryAt ? new Date(interruption.nextRetryAt) : null, continuedBySweep: strandedHoldContinuesAutomatically(action, strandedRuns.get(String(action.evidence?.latestRunId ?? "")) ?? null, issue.assigneeAgentId, at) } : null, - agent: agent ? { status: agent.status, invokable: agent.invokable } : null, + agent: agent ? { status: agent.status, invokable: agent.invokable, updatedAt: agent.updatedAt } : null, companyPaused: company?.status === "paused", pauseHold, budgetStop: companyBudget || agent?.pauseReason === "budget" || agent?.pauseReason === "budget_exceeded", @@ -582,14 +587,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } /** Evaluate one task, and repair it when it violates the invariant. */ - async function checkIssue(scope: { companyId: string; issueId: string; onDeferredDeadline?: (at: number) => void }): Promise { + async function checkIssue(scope: { companyId: string; issueId: string; + onDeferredDeadline?: (at: number, detail?: { unchanged: boolean }) => void }): Promise { const [issue] = await loadIssues(scope.companyId, scope.issueId, 1); if (!issue) return null; const facts = (await loadFacts(scope.companyId, [issue])).get(issue.id)!; const at = now(); const verdict = evaluateAssignedWork(facts, at); const check = await act(issue, facts, verdict, at); - if (check.retryAt) scope.onDeferredDeadline?.(check.retryAt.getTime()); + if (check.retryAt) scope.onDeferredDeadline?.(check.retryAt.getTime(), { unchanged: check.outcome === "waiting_unchanged" }); passLog.record(check); return check; } @@ -639,6 +645,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { retryAt: new Date(at.getTime() + CONTINUATION_CAP_WINDOW_MS) }; } } + const key = stateKey(facts, verdict); + const refusals = await refusalsInState(issue, key, at); + if (refusals.count && refusals.lastAt !== null) { + const waitUntil = refusals.lastAt + Math.min(UNCHANGED_REFUSAL_MAX_WAIT_MS, CONTINUATION_BASE_BACKOFF_MS * 2 ** (refusals.count - 1)); + if (waitUntil > at.getTime()) return { verdict, outcome: "waiting_unchanged", retryAt: new Date(waitUntil) }; + } + const refusal = { stateKey: key, refusedAt: at.toISOString(), + ...(refusals.count ? { message: CONTINUATION_REFUSED_AGAIN_MESSAGE } : {}) }; // A small process-wide budget keeps a post-deploy backlog from starting // hundreds of runs at once. Deferred tasks come back on their deadline. const budgetDeferral = repairBudgetDeferral(at); @@ -650,7 +664,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit", retryAt: claimed.kind === "changed" ? null : claimed.retryAt }; repairTimes.push(at.getTime()); - if (claimed.reason === "queued_run_stalled") return replaceStalledRun(issue, claimed, claimed.verdict.runId!); + if (claimed.reason === "queued_run_stalled") return replaceStalledRun(issue, claimed, claimed.verdict.runId!, refusal); // A block that interrupted a review goes back to review; the review's own // recovery wakes its participant. No agent work starts from here. if (claimed.restoredTo === "in_review") { @@ -694,10 +708,31 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (!run) await release(); if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", - fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}); + { ...(fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}), ...(run ? {} : refusal) }); return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } + function stateKey(facts: AssignedWorkFacts, verdict: Extract) { + const { updatedAt: _updatedAt, ...issue } = facts.issue; + return createHash("sha256").update(JSON.stringify({ reason: verdict.reason, ...facts, issue })).digest("hex").slice(0, 32); + } + + async function refusalsInState(issue: IssueRow, key: string, at: Date) { + const rows = await db.select({ details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.companyId, issue.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), + gte(activityLog.createdAt, new Date(at.getTime() - UNCHANGED_REFUSAL_CHAIN * UNCHANGED_REFUSAL_MAX_WAIT_MS)), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))).orderBy(desc(activityLog.createdAt)).limit(UNCHANGED_REFUSAL_CHAIN); + let count = 0, lastAt: number | null = null; + for (const row of rows) { + const details = json(row.details); + if (details.outcome !== "not_started" || details.stateKey !== key) break; + count += 1; + const refusedAt = Date.parse(String(details.refusedAt)); + if (lastAt === null && Number.isFinite(refusedAt)) lastAt = refusedAt; + } + return { count, lastAt }; + } + function repairBudgetDeferral(at: Date): Date | null { const minuteAgo = at.getTime() - 60_000; while (repairTimes.length && repairTimes[0] <= minuteAgo) repairTimes.shift(); @@ -715,7 +750,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { "queued run its agent keeps passing over was left in place"); } - async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string): Promise { + async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string, refusal: Record): Promise { const slot = await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: runId }); if (slot.heldUntil) { await settleContinuationActivity(claimed, claimed.reason, "not_started", @@ -743,7 +778,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return null; }); if (!run) await slot.release(); - await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); + await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", run ? {} : refusal); return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 6fae89d60e0..fc7b39b628b 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -10,7 +10,7 @@ import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true }; -type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void }; +type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void }; const PAGE_SIZE = 100; export const resolvedDependencyCandidate = sql`exists (select 1 from ${issueRelations} r where r.company_id = ${issues.companyId} @@ -126,7 +126,7 @@ export function registerIssueRecoveryWork( * company dirty; checks schedule their own grace and back-off deadlines. * Newest-updated tasks are checked first, a small batch per pass. */ function registerAssignedWork(sweep: NonNullable) { - type Suspect = Entry & { updatedAt: number }; + type Suspect = Entry & { updatedAt: number; unchanged?: true; fromBackstop?: true }; const cursors = new Map(); const passes = fullPasses(); let enabled = false; @@ -139,7 +139,8 @@ export function registerIssueRecoveryWork( decodeScope: decodeIssueRecoveryHint, matches: issueRecoveryScopeMatches, priority: entry => entry.updatedAt, - mergeEntry: (pending, loaded) => ({ ...loaded, dueAt: Math.max(pending.dueAt, loaded.dueAt) }), + mergeEntry: (pending, loaded) => pending.unchanged && !loaded.fromBackstop ? loaded + : { ...loaded, dueAt: Math.max(pending.dueAt, loaded.dueAt), ...(pending.unchanged ? { unchanged: true as const } : {}) }, load: async scope => { const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } @@ -149,14 +150,16 @@ export function registerIssueRecoveryWork( const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); passes.loaded(scope, complete); - return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now() })), complete }; + return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), + ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete }; }, check: async entry => { if (!enabled) return null; - let next: number | null = null; + let next: number | null = null, unchanged = false; await sweep.checkIssue({ companyId: entry.companyId, issueId: entry.id, - onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); - return next === null ? null : { ...entry, dueAt: next }; + onDeferredDeadline: (at, detail) => { next = next === null ? at : Math.min(next, at); unchanged ||= detail?.unchanged === true; } }); + const { unchanged: _previous, fromBackstop: _source, ...rest } = entry; + return next === null ? null : { ...rest, dueAt: next, ...(unchanged ? { unchanged: true as const } : {}) }; }, }, canRun); return worker;