diff --git a/server/src/__tests__/native-failed-run-owner-release-idle.test.ts b/server/src/__tests__/native-failed-run-owner-release-idle.test.ts new file mode 100644 index 00000000000..eb6d173a0ff --- /dev/null +++ b/server/src/__tests__/native-failed-run-owner-release-idle.test.ts @@ -0,0 +1,134 @@ +import { randomUUID } from "node:crypto"; +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js"; +import { executionControlSchedule, registerExecutionControlWork } from "../services/execution-control-work.js"; +import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js"; +import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; +import { nextAbandonedNativeOwnerReleaseAt, resetAbandonedNativeOwnerReleaseForTests } from "../services/native-runtime/native-failed-run-owner-release.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +describeIssueRecovery("failed native runs held by an unverifiable foreign controller", () => { + const state = issueRecoveryFixture(); + + async function seedTask(f: { companyId: string; agentId: string }, status: "in_progress" | "done" | "cancelled") { + const db = state.db; + const issueId = randomUUID(), runId = randomUUID(); + const leaseExpiresAt = new Date(Date.now() - 40 * 60_000); + await db.insert(issues).values({ id: issueId, companyId: f.companyId, title: `Native task ${status}`, status, + assigneeAgentId: f.agentId, workMode: "standard", executionRunId: null, checkoutRunId: null, + ...(status === "done" ? { completedAt: new Date() } : status === "cancelled" ? { cancelledAt: new Date() } : {}) }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: issueId, + status: "failed", runtimeMode: "native", runtimeModeResolvedAt: new Date(), contextSnapshot: { issueId }, + startedAt: new Date(Date.now() - 60 * 60_000), finishedAt: new Date(Date.now() - 50 * 60_000), errorCode: "setup_failed" }); + await db.insert(nativeRunFinalizations).values({ runId, companyId: f.companyId, issueId, phase: "observed", attempt: 1, + leaseOwner: `foreign:${randomUUID()}`, leaseExpiresAt, controllerBootId: randomUUID(), controllerHostHash: null, + recoveryState: "awaiting_evidence" }); + if (status !== "in_progress") { + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", ownerType: "board", returnOwnerAgentId: f.agentId, cause: "native_restart_recovery_blocked", + fingerprint: `native-controller-space:${runId}:${"0".repeat(64)}`, attemptCount: 0, outcome: "restored", + resolutionNote: "source_terminal", resolvedAt: new Date(leaseExpiresAt.getTime() - 60_000), + evidence: { runId, reason: "controller_process_space_unverified", executorStopEvidence: "unverified" }, + nextAction: "The server running this task restarted. Paperclip continues the task in a fresh run from its saved work shortly." }); + } + return { issueId, runId }; + } + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(500, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + async function measure(work: () => Promise) { + const statements: string[] = [], signals: string[] = [], prior = state.db.$client.options.debug; + const unsubscribes = Object.values(DELIVERY_QUEUES).map(queue => subscribeDatabaseWork(state.db, queue, event => { + if (event === "settled") signals.push(queue); + })); + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { await work(); } finally { + state.db.$client.options.debug = prior; + for (const unsubscribe of unsubscribes) unsubscribe(); + } + return { + statements: statements.length, + attentionInserts: statements.filter(query => query.startsWith("insert into \"issue_recovery_actions\"")).length, + attentionWrites: statements.filter(query => /^(insert into|update) "issue_recovery_actions"/.test(query)).length, + runWrites: statements.filter(query => query.startsWith("update \"heartbeat_runs\"")).length, + signals: signals.length, + }; + } + + async function start() { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + resetAbandonedNativeOwnerReleaseForTests(); + await state.start({}); + const schedule = executionControlSchedule(state.db); + await registerExecutionControlWork(state.coordinator, [ + { queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(state.db, new Date()), + next: schedule.finalization }, + { queue: DELIVERY_QUEUES.executionDisposition, run: () => settleUnrecoverableExecutions(state.db), next: schedule.disposition }, + ]).ready; + } + + const rowsFor = (issueId: string) => state.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + + it("stays quiet after a live task's surfaced attention is settled", async () => { + const f = await state.fixture(); + await state.db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId)); + const tasks = [await seedTask(f, "in_progress"), await seedTask(f, "in_progress"), await seedTask(f, "in_progress")]; + await start(); + for (let minute = 0; minute < 30; minute++) { + await advance(60_000); + const active = await state.db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, f.companyId), eq(issueRecoveryActions.status, "active"))); + const resolved = await state.db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, f.companyId), eq(issueRecoveryActions.status, "resolved"))); + if (active.length === 0 && resolved.length === tasks.length) break; + } + for (const task of tasks) { + expect(await rowsFor(task.issueId)).toEqual([expect.objectContaining({ status: "resolved", + resolutionNote: "continued_fresh_from_last_saved", fingerprint: expect.stringContaining(`native-controller-space:${task.runId}:`) })]); + } + expect(await nextAbandonedNativeOwnerReleaseAt(state.db)).toBeNull(); + const idle = await measure(async () => { + for (let wake = 0; wake < 6; wake++) { + await state.db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization)); + await advance(5 * 60_000); + } + }); + expect({ attentionWrites: idle.attentionWrites, runWrites: idle.runWrites, signals: idle.signals }) + .toEqual({ attentionWrites: 0, runWrites: 0, signals: 6 }); + expect(idle.statements).toBeLessThan(6 * 60); + expect(await state.db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId), + eq(issueRecoveryActions.status, "active")))).toEqual([]); + }, 300_000); + + it("never surfaces attention for finished tasks and surfaces a live task exactly once", async () => { + const f = await state.fixture(); + await state.db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId)); + const done = await seedTask(f, "done"); + const cancelled = await seedTask(f, "cancelled"); + const live = await seedTask(f, "in_progress"); + const idle = await measure(async () => { + await start(); + await advance(30 * 60_000); + }); + for (const task of [done, cancelled]) { + expect(await rowsFor(task.issueId)).toEqual([expect.objectContaining({ status: "resolved", resolutionNote: "source_terminal" })]); + } + const liveRows = await rowsFor(live.issueId); + expect(liveRows).toEqual([expect.objectContaining({ cause: "native_restart_recovery_blocked", + fingerprint: expect.stringContaining(`native-controller-space:${live.runId}:`), + evidence: expect.objectContaining({ runId: live.runId, reason: "controller_process_space_unverified" }) })]); + expect(idle.attentionInserts).toBe(1); + expect(await nextAbandonedNativeOwnerReleaseAt(state.db)).toBeNull(); + expect(await state.db.select({ phase: nativeRunFinalizations.phase }).from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, done.runId))).toEqual([{ phase: "observed" }]); + }, 300_000); +}); diff --git a/server/src/services/native-runtime/native-failed-run-owner-release.test.ts b/server/src/services/native-runtime/native-failed-run-owner-release.test.ts index 146aa9585b8..716cfd9d3ed 100644 --- a/server/src/services/native-runtime/native-failed-run-owner-release.test.ts +++ b/server/src/services/native-runtime/native-failed-run-owner-release.test.ts @@ -361,6 +361,50 @@ describe("failed native run owner release", () => { expect(await nextAbandonedNativeOwnerReleaseAt(db)).toBeNull(); }); + it.each(["done", "cancelled"] as const)("never surfaces namespace attention for a %s task until it is reopened", async status => { + const { runId, issueId } = await nativeRun({ status: "failed", coordinator: { + leaseOwner: `foreign:${randomUUID()}`, controllerHostHash: null, leaseExpiresAt: new Date(0) } }); + await db.update(issues).set({ status }).where(eq(issues.id, issueId)); + const before = await coordinator(runId); + expect(await nextAbandonedNativeOwnerReleaseAt(db)).toBeNull(); + for (let pass = 0; pass < 3; pass++) { + expect(await committedWakes(DELIVERY_QUEUES.strandedAssignments, () => releaseAbandonedNativeOwners(db))).toBe(0); + } + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toEqual([]); + expect(await coordinator(runId)).toEqual(before); + expect(await executionControlSchedule(db).finalization()).toBeNull(); + await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, issueId)); + expect(await nextAbandonedNativeOwnerReleaseAt(db)).not.toBeNull(); + expect(await releaseAbandonedNativeOwners(db)).toBe(0); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toEqual([ + expect.objectContaining({ status: "active", evidence: expect.objectContaining({ runId }) })]); + }); + + it("keeps a settled namespace incident settled and surfaces a renewed authority once", async () => { + const { runId, issueId } = await nativeRun({ status: "failed", coordinator: { + leaseOwner: `foreign:${randomUUID()}`, controllerHostHash: null, leaseExpiresAt: new Date(Date.now() - 60_000) } }); + expect(await releaseAbandonedNativeOwners(db)).toBe(0); + const [surfaced] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(surfaced).toMatchObject({ status: "active", evidence: { runId } }); + await db.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: new Date(), + resolutionNote: "source_terminal" }).where(eq(issueRecoveryActions.id, surfaced.id)); + expect(await nextAbandonedNativeOwnerReleaseAt(db)).toBeNull(); + for (let pass = 0; pass < 3; pass++) { + expect(await committedWakes(DELIVERY_QUEUES.strandedAssignments, () => releaseAbandonedNativeOwners(db))).toBe(0); + } + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(1); + const renewedExpiry = new Date(Date.now() + 60_000); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: renewedExpiry }).where(eq(nativeRunFinalizations.runId, runId)); + expect((await nextAbandonedNativeOwnerReleaseAt(db))?.getTime()).toBe(renewedExpiry.getTime()); + const afterRenewal = new Date(renewedExpiry.getTime() + 1); + expect(await releaseAbandonedNativeOwners(db, afterRenewal)).toBe(0); + expect(await releaseAbandonedNativeOwners(db, afterRenewal)).toBe(0); + const rows = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(rows).toHaveLength(2); + expect(rows.find(row => row.id !== surfaced.id)).toMatchObject({ status: "active", evidence: { runId } }); + expect(rows.find(row => row.id !== surfaced.id)?.fingerprint).not.toBe(surfaced.fingerprint); + }); + it("keeps an unexpired owner blocking the stop check and schedules its expiry", async () => { const leaseExpiresAt = new Date(Date.now() + 10 * 60_000); const { runId } = await nativeRun({ status: "failed", errorCode: "setup_failed", error: LEASE_FAILURE, diff --git a/server/src/services/native-runtime/native-failed-run-owner-release.ts b/server/src/services/native-runtime/native-failed-run-owner-release.ts index 172d40b84a2..88bdbf934d8 100644 --- a/server/src/services/native-runtime/native-failed-run-owner-release.ts +++ b/server/src/services/native-runtime/native-failed-run-owner-release.ts @@ -15,7 +15,7 @@ import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { LEGACY_WORKSPACE_RECOVERY_KEY } from "../workspace-restore-recovery-state.js"; import { activeNativeExecutionRunIds } from "./native-active-runs.js"; -import { currentNativeControllerIdentity, isCurrentNativeControllerProcess, surfaceNativeControllerNamespaceAttention, recordLegacyHostFallbackProof } from "./native-restart-recovery.js"; +import { currentNativeControllerIdentity, isCurrentNativeControllerProcess, settledNativeControllerNamespaceAttention, surfaceNativeControllerNamespaceAttention, recordLegacyHostFallbackProof } from "./native-restart-recovery.js"; import { nativeControllerOwnerStamp, runProcessOwnerStamp } from "./native-owner-stamps.js"; /** @@ -60,6 +60,13 @@ const unsettledRunEnvironmentLease = sql`exists ( /** A pending Stop belongs to native restart recovery's stop cleanup. */ const pendingNativeStop = sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) #>> '{nativeCancellation,dispatchState}' = 'pending'`; +const quietNamespaceOwner = () => sql`(${settledNativeControllerNamespaceAttention({ + companyId: nativeRunFinalizations.companyId, issueId: nativeRunFinalizations.issueId, + runId: nativeRunFinalizations.runId, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, +})} or exists (select 1 from ${issues} source_task + where source_task.company_id = ${nativeRunFinalizations.companyId} and source_task.id = ${nativeRunFinalizations.issueId} + and source_task.status in ('done', 'cancelled')))`; + /** * Rows the background pass may release once their lease expires. A pinned * workspace restore belongs to observed-workspace preservation, which must stop @@ -108,6 +115,7 @@ export async function nextAbandonedNativeOwnerReleaseAt(db: Db): Promise`extract(epoch from ${nativeRunFinalizations.leaseExpiresAt})::numeric::text` }) .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); const { remoteProcess } = await readNativeRemoteExecutionOwnership(tx, { run, issueId: coordinator.issueId }); @@ -379,6 +388,7 @@ export async function releaseAbandonedNativeOwners(db: Db, now = new Date(), pag // One bounded page per pass. The cursor rotates across passes, so rows that a // live process still holds cannot starve stopped owners behind them; any // remaining expired row keeps the finalization deadline due for the next pass. + const local = await localOwnerSpaceCondition(); const page = (after: string | null) => db.select({ runId: nativeRunFinalizations.runId, leaseOwner: nativeRunFinalizations.leaseOwner, @@ -388,6 +398,7 @@ export async function releaseAbandonedNativeOwners(db: Db, now = new Date(), pag .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) .where(and(abandonedOwnerCondition(), lte(nativeRunFinalizations.leaseExpiresAt, now), + sql`(coalesce(${local}, false) or not ${quietNamespaceOwner()})`, after ? gt(nativeRunFinalizations.runId, after) : undefined)) .orderBy(asc(nativeRunFinalizations.runId)) .limit(pageSize + 1); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 066de14ceda..0eb12f5a925 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -26,7 +26,7 @@ import { recordNativeLocalProcessStopInTransaction, } from "../native-local-process-stop.js"; import { randomUUID } from "node:crypto"; -import { and, desc, eq, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from "drizzle-orm"; +import { and, desc, eq, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql, type SQLWrapper } from "drizzle-orm"; import { alias } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { @@ -1062,6 +1062,17 @@ async function endNamespaceUnverifiedRun(tx: Db, row: { if (updatedRun) await preserveNativeFailedWorkspace(tx, updatedRun); } +export function settledNativeControllerNamespaceAttention(owner: { + companyId: SQLWrapper | string; issueId: SQLWrapper | string; runId: SQLWrapper | string; leaseExpiresAt: SQLWrapper | string; +}) { + return sql`exists (select 1 from ${issueRecoveryActions} settled + where settled.company_id = ${owner.companyId}::uuid and settled.source_issue_id = ${owner.issueId}::uuid + and settled.evidence->>'runId' = ${owner.runId}::text + and settled.status = 'resolved' and settled.cause = 'native_restart_recovery_blocked' + and starts_with(settled.fingerprint, 'native-controller-space:' || ${owner.runId}::text || ':') + and settled.resolved_at >= ${owner.leaseExpiresAt}::timestamptz)`; +} + /** Shared by restart inspection and failed-owner cleanup under their existing * issue/coordinator/run locks. Publishing attention never grants a new turn. */ export async function surfaceNativeControllerNamespaceAttention(db: Db, input: { @@ -1085,6 +1096,12 @@ export async function surfaceNativeControllerNamespaceAttention(db: Db, input: { const sameSource = existing?.cause === "native_restart_recovery_blocked" && existing.evidence.runId === input.run.id && ["controller_process_space_unverified", "remote_controller_process_space_unverified"].includes(String(existing.evidence.reason)); + if (!existing && input.coordinator.leaseExpiresAt) { + const [settled] = await db.execute<{ settled: boolean }>(sql`select ${settledNativeControllerNamespaceAttention({ + companyId: input.run.companyId, issueId: input.coordinator.issueId, runId: input.run.id, + leaseExpiresAt: input.coordinator.leaseExpiresAt.toISOString() })} as settled`); + if (settled?.settled) return; + } if (!existing || (sameSource && existing.fingerprint !== fingerprint)) { await attention.upsertSourceScoped({ companyId: input.run.companyId, sourceIssueId: input.coordinator.issueId, @@ -1491,7 +1508,7 @@ export async function claimNativeRestartRecoveries(input: { } } if (!takeover.allowed) { - if (namespaceAttentionNeeded) { + if (namespaceAttentionNeeded && !["done", "cancelled"].includes(row.issueStatus)) { await surfaceNativeControllerNamespaceAttention(tx as unknown as Db, { run: row.run, coordinator: row.coordinator, expiryEpoch: row.controllerLeaseExpiryEpoch, remoteProcess, });