diff --git a/server/src/__tests__/continuation-delivery-unwakeable-agent-idle.test.ts b/server/src/__tests__/continuation-delivery-unwakeable-agent-idle.test.ts new file mode 100644 index 00000000000..e6b41276ca4 --- /dev/null +++ b/server/src/__tests__/continuation-delivery-unwakeable-agent-idle.test.ts @@ -0,0 +1,139 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { eq } from "drizzle-orm"; +import { afterEach, expect, it, vi } from "vitest"; +import { agentWakeupRequests, agents, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { configureAgentLifecycle, createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { executionControlSchedule, registerExecutionControlWork } from "../services/execution-control-work.js"; +import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; +import { deferForUnwakeableAssignee, readmitAgentContinuations } from "../services/recovery/continuation-delivery-attempt.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +describeIssueRecovery("continuation delivery to an agent that cannot be woken", () => { + const state = issueRecoveryFixture(); + const passes = new Set>(); + let lifecycleWorker: ReturnType | undefined; + const lifecycle = () => lifecycleWorker ??= configureAgentLifecycle(state.db, + { requiredPluginIds: async () => [], runHost: async () => "complete", runPlugin: async () => "complete" }); + afterEach(async () => { await lifecycleWorker?.stop(); lifecycleWorker = undefined; }); + + async function seed(status: "paused" | "terminated" | "pending_approval") { + const f = await state.fixture(); + const db = state.db; + if (status === "paused") { await createAgentLifecycle(db).pauseAgent(f.agentId); await lifecycle().process(f.agentId); } + else await db.update(agents).set({ status }).where(eq(agents.id, f.agentId)); + expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0]!.status).toBe(status); + const [task] = await db.insert(issues).values({ companyId: f.companyId, title: "Continue the work", status: "in_progress", + assigneeAgentId: f.agentId }).returning(); + const [source] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "failed", + errorCode: "adapter_failed", startedAt: new Date(Date.now() - 3 * MINUTE), finishedAt: new Date(Date.now() - 2 * MINUTE), + contextSnapshot: { issueId: task!.id } }).returning(); + const recordedAt = new Date().toISOString(); + const [action] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task!.id, + kind: "active_run_watchdog", status: "resolved", outcome: "restored", resolvedAt: new Date(), ownerType: "system", + returnOwnerAgentId: f.agentId, cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${source!.id}`, + nextAction: "Continue the task.", attemptCount: 1, + evidence: { runId: source!.id, originalFailureCode: "adapter_failed", continuationDelivery: "pending", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", sourceRunId: source!.id, trigger: "recovery_refused", + delivery: "recorded", recordedAt, isolation: "writer_ended", stopProof: "writer_ended" }, + freshContinuationSettlement: { recordedAt }, + executionReconciliation: { runId: source!.id, providerStopped: true, providerStopProof: "writer_ended", + actionOutcome: "mixed", outcomeEvidence: "The run stopped; the continuation starts from the last saved work.", + actorId: "fresh-continuation", recordedAt } } }).returning(); + return { ...f, taskId: task!.id, actionId: action!.id }; + } + + async function start(queueOnly = false) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, canReconcile: () => true, + onError: error => state.errors.push(error) }); + state.coordinator = coordinator; + const wake = vi.fn(async (agentId: string, opts: Parameters[1]) => { + if (!queueOnly) return state.heartbeat.wakeup(agentId, opts); + const [agent] = await state.db.select().from(agents).where(eq(agents.id, agentId)); + const [run] = await state.db.insert(heartbeatRuns).values({ companyId: agent!.companyId, agentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts?.contextSnapshot ?? {} }).returning(); + return run!; + }); + const work = registerExecutionControlWork(coordinator, [{ queue: DELIVERY_QUEUES.executionContinuation, run: () => { + const pass = deliverReconciledExecutions(state.db, wake as never); passes.add(pass); + void pass.then(() => passes.delete(pass), () => passes.delete(pass)); + return pass; + }, next: executionControlSchedule(state.db).continuation }]); + await work.ready; + return wake; + } + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1_000, until - Date.now())); + await Promise.allSettled([...passes]); await immediate(); + } + } + + async function countStatements(work: () => Promise) { + const statements: string[] = [], prior = state.db.$client.options.debug; + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { await work(); } finally { state.db.$client.options.debug = prior; } + return statements; + } + + const evidence = async (id: string) => + (await state.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!.evidence; + + for (const status of ["paused", "terminated", "pending_approval"] as const) { + it(`waits instead of retrying every pass while the assignee is ${status}`, async () => { + const f = await seed(status); + const wake = await start(); + await advance(MINUTE); + const idle = await countStatements(() => advance(2 * HOUR)); + expect(wake).not.toHaveBeenCalled(); + expect(idle.length).toBeLessThan(300); + expect(await evidence(f.actionId)).toMatchObject({ continuationDelivery: "pending", + continuationDeliveryAgentWaits: expect.any(Number) }); + expect(await state.db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agentId))).toHaveLength(0); + const [task] = await state.db.select().from(issues).where(eq(issues.id, f.taskId)); + expect(task).toMatchObject({ status: "in_progress", assigneeAgentId: f.agentId }); + }, 600_000); + } + + it("delivers the continuation within seconds once the paused assignee is resumed", async () => { + const f = await seed("paused"); + const wake = await start(true); + await advance(3 * HOUR); + expect(wake).not.toHaveBeenCalled(); + await createAgentLifecycle(state.db).resumeAgent(f.agentId); + await lifecycle().process(f.agentId); + expect((await state.db.select().from(agents).where(eq(agents.id, f.agentId)))[0]!.status).not.toBe("paused"); + await advance(20_000); + expect(wake).toHaveBeenCalledTimes(1); + expect(await evidence(f.actionId)).toMatchObject({ continuationDelivery: "delivered", continuationRunId: expect.any(String) }); + }, 600_000); + + it("does not store a wait when the assignee is resumed while delivery is checking it", async () => { + const f = await seed("terminated"); + const [action] = await state.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId)); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let locked!: () => void; + const lockTaken = new Promise(resolve => { locked = resolve; }); + const resume = state.db.transaction(async tx => { + await tx.update(agents).set({ status: "idle" }).where(eq(agents.id, f.agentId)); + await readmitAgentContinuations(tx as never, { companyId: f.companyId, agentId: f.agentId }); + locked(); + await held; + }); + await lockTaken; + const deferred = deferForUnwakeableAssignee(state.db, action!); + await new Promise(resolve => setTimeout(resolve, 200)); + release(); + await resume; + expect(await deferred).toBe(false); + expect(await evidence(f.actionId)).not.toHaveProperty("continuationDeliveryRetryAt"); + }, 60_000); +}); diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 619491c22d1..e38b30280d4 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,5 +1,6 @@ import { notifyMuseContactVerification } from "../modules/external-agents/index.js"; import { notifyBudgetResumeAgentReady } from "./budget-resume-continuation.js"; +import { readmitAgentContinuations } from "./recovery/continuation-delivery-attempt.js"; import { currentControllerProcessStamp } from "./controller-process-identity.js"; import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -70,6 +71,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li await notifyNativeReviewResponse(tx, { companyId, agentId }); await notifyChatInboundWakeNotices(tx, { companyId }); await notifyBudgetResumeAgentReady(tx, agentId); + await readmitAgentContinuations(tx, { companyId, agentId }); } }, reportFailure(context, error) { diff --git a/server/src/services/execution-control-work.ts b/server/src/services/execution-control-work.ts index 8bc16172fb9..eb3e1983964 100644 --- a/server/src/services/execution-control-work.ts +++ b/server/src/services/execution-control-work.ts @@ -10,6 +10,7 @@ import { hasPendingNativeReplacements, nextNativeOwnershipProofDeadlineAt } from import { nextAbandonedNativeOwnerReleaseAt } from "./native-runtime/native-failed-run-owner-release.js"; import { hasPendingPauseResume } from "./issue-tree-resume.js"; import { hasPendingBudgetResume } from "./budget-resume-continuation.js"; +import { continuationDeliveryDue } from "./recovery/continuation-delivery-attempt.js"; export function registerExecutionControlWork( coordinator: ReturnType, @@ -94,11 +95,16 @@ export function executionControlSchedule(db: Db) { return at === null ? null : deadline([new Date(at)]); }, async continuation() { - const rows = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + const pending = and( eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - )).limit(1); - return rows.length || await hasPendingPauseResume(db) || await hasPendingBudgetResume(db) ? retry() : null; + ); + const rows = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(and(pending, continuationDeliveryDue(new Date()))).limit(1); + if (rows.length || await hasPendingPauseResume(db) || await hasPendingBudgetResume(db)) return retry(); + const [waiting] = await db.select({ at: sql`min((${issueRecoveryActions.evidence}->>'continuationDeliveryRetryAt')::timestamptz)` }) + .from(issueRecoveryActions).where(pending); + return deadline([waiting?.at ? new Date(waiting.at) : null]); }, async status() { const rows = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 250c985095d..d3dc9e68a94 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -40,7 +40,7 @@ import { readSystemRecovery } from "./system-recovery-state.js"; import { isReattachPhaseActive } from "./heartbeat/run-admission.js"; import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; import { reserveAutomaticRunCreation } from "./recovery/automatic-run-brake.js"; -import { recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js"; +import { continuationDeliveryDue, deferForUnwakeableAssignee, recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; @@ -423,6 +423,7 @@ export async function deliverReconciledExecutions( and( eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + continuationDeliveryDue(new Date()), seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, seenAgents.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgents])}::text[])` : undefined, companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, @@ -545,6 +546,7 @@ export async function deliverReconciledExecutions( continue; } } + if (await deferForUnwakeableAssignee(db, action)) { seenAgents.add(action.returnOwnerAgentId); continue; } if (!await agentHasRoom(action, action.returnOwnerAgentId, kind)) { seenAgents.add(action.returnOwnerAgentId); continue; diff --git a/server/src/services/recovery/continuation-delivery-attempt.ts b/server/src/services/recovery/continuation-delivery-attempt.ts index ce3b7c2d221..c1c2d78668a 100644 --- a/server/src/services/recovery/continuation-delivery-attempt.ts +++ b/server/src/services/recovery/continuation-delivery-attempt.ts @@ -1,15 +1,69 @@ import { and, eq, sql } from "drizzle-orm"; -import { issueRecoveryActions, type Db } from "@paperclipai/db"; +import { agents, issueRecoveryActions, type Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; +import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; + +export const CONTINUATION_AGENT_WAIT_BASE_MS = 60_000; +export const CONTINUATION_AGENT_WAIT_MAX_MS = 60 * 60_000; + +type PendingAction = Pick; + +const pendingDelivery = (action: PendingAction) => and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, +); /** Records a delivery attempt for a pending continuation, so the next pass tries the records not yet tried first. */ -export function recordContinuationDeliveryAttempt(db: Db, action: Pick) { +export function recordContinuationDeliveryAttempt(db: Db, action: PendingAction) { return db.update(issueRecoveryActions) .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) - .where(and( - eq(issueRecoveryActions.companyId, action.companyId), - eq(issueRecoveryActions.id, action.id), - eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - )); + .where(pendingDelivery(action)); +} + +export function continuationAgentWait(evidence: Record, now = new Date()) { + const recorded = Number(evidence.continuationDeliveryAgentWaits ?? 0); + const prior = Number.isSafeInteger(recorded) && recorded > 0 ? recorded : 0; + const delay = Math.min(CONTINUATION_AGENT_WAIT_MAX_MS, CONTINUATION_AGENT_WAIT_BASE_MS * 2 ** Math.min(prior, 20)); + return { waits: prior + 1, retryAt: new Date(now.getTime() + delay) }; +} + +export async function deferForUnwakeableAssignee(db: Db, + action: PendingAction & Pick, now = new Date()) { + if (!action.returnOwnerAgentId) return false; + return db.transaction(async tx => { + const [owner] = await tx.select().from(agents).where(and(eq(agents.companyId, action.companyId), + eq(agents.id, action.returnOwnerAgentId!))).for("share"); + if (!owner || (await evaluateAgentInvokabilityFromDb(tx as unknown as Db, + isAgentAwaitingSetup(owner) ? { ...owner, status: "idle" } : owner)).invokable) return false; + const { waits, retryAt } = continuationAgentWait(action.evidence, now); + await tx.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ continuationDeliveryAttemptedAt: now.toISOString(), + continuationDeliveryRetryAt: retryAt.toISOString(), continuationDeliveryAgentWaits: waits })}::jsonb` }) + .where(pendingDelivery(action)); + return true; + }); +} + +export function continuationDeliveryDue(now: Date) { + return sql`coalesce((${issueRecoveryActions.evidence}->>'continuationDeliveryRetryAt')::timestamptz <= ${now.toISOString()}::timestamptz, true)`; +} + +export async function readmitAgentContinuations(tx: Db, input: { companyId: string; agentId: string }) { + const waiting = and( + eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.returnOwnerAgentId, input.agentId), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + sql`${issueRecoveryActions.evidence} ? 'continuationDeliveryRetryAt'`, + ); + const [any] = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(waiting).limit(1); + if (!any) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); + await tx.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} - 'continuationDeliveryRetryAt' - 'continuationDeliveryAgentWaits'` }) + .where(waiting); }