diff --git a/server/src/__tests__/heartbeat-lease-release-before-runner-attach.test.ts b/server/src/__tests__/heartbeat-lease-release-before-runner-attach.test.ts new file mode 100644 index 00000000000..f1f5229d7eb --- /dev/null +++ b/server/src/__tests__/heartbeat-lease-release-before-runner-attach.test.ts @@ -0,0 +1,285 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + companies, + createDb, + environmentLeases, + environments, + heartbeatRunEvents, + heartbeatRuns, + issues, + nativeRunFinalizations, + plugins, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); +vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); + +import { heartbeatService } from "../services/heartbeat.ts"; +import { createLifecycleDriver } from "../services/agent-lifecycle-driver.js"; +import { configureAgentLifecycle, createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "../services/native-local-process-stop.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = support.supported ? describe : describe.skip; + +type Window = "artifact_discovery" | "provider_pack_select" | "provider_pack_upload" | "runner_launch" | "transport_connect" | "session_open"; +const PRE_LAUNCH: Window[] = ["artifact_discovery", "provider_pack_select", "provider_pack_upload"]; +const ALL_WINDOWS: Window[] = [...PRE_LAUNCH, "runner_launch", "transport_connect", "session_open"]; +const PROVIDER_METHODS = ["environmentAcquireLease", "environmentExecute", "environmentReleaseLease", + "environmentDestroyLease", "environmentStopLease"]; + +describeEmbeddedPostgres("environment lease release between sandbox acquire and runner attach", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + const workers: Array<{ stop(): Promise }> = []; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-lease-release-before-attach-"); + db = createDb(tempDb.connectionString); + }, 30_000); + + afterEach(async () => { + for (const worker of workers.splice(0)) await worker.stop(); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + function providerWorker(pluginId: string) { + const call = vi.fn(async (_pluginId: string, method: string, params: { providerLeaseId?: string }) => { + if (method === "environmentDestroyLease") return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; + if (method === "environmentStopLease") return { providerLeaseId: params.providerLeaseId, state: "stopped" }; + if (method === "environmentReleaseLease") return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; + throw new Error(`Unexpected provider method ${method}`); + }); + const manager = { + isRunning: vi.fn((id: string) => id === pluginId), + getWorker: vi.fn(() => ({ supportedMethods: PROVIDER_METHODS })), + call, + } as unknown as PluginWorkerManager; + const teardownCalls = () => call.mock.calls.filter(([, method]) => + method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease"); + return { manager, teardownCalls }; + } + + async function seed(window: Window) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const environmentId = randomUUID(), leaseId = randomUUID(), pluginId = randomUUID(); + const providerLeaseId = `sandbox-${randomUUID()}`; + const pluginKey = `paperclip.lease-release-${pluginId.slice(0, 8)}`; + await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", + apiVersion: 1, categories: ["automation"], status: "ready", installOrder: 1, updatedAt: new Date(), + manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Fake sandbox", description: "Test provider", + author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [{ driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", + configSchema: { type: "object" } }] } } as never); + await db.insert(companies).values({ id: companyId, name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "OpenCoder", role: "engineer", status: "running", + adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" }, runtimeConfig: {}, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, title: "Short task", status: "in_progress", + priority: "medium", assigneeAgentId: agentId }); + const config = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false }; + await db.insert(environments).values({ id: environmentId, name: `Fake sandbox ${environmentId}`, driver: "sandbox", + status: "active", config }); + const startedAt = new Date(Date.now() - 10_000); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", status: "running", + runtimeMode: "native", runtimeModeResolvedAt: startedAt, issueId, nativeIssueId: issueId, startedAt, + contextSnapshot: { issueId }, nextEventSeq: 1, + runnerProfileJson: { nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId, providerLeaseId, + remoteCwd: "/workspace", resourceDisposition: "destroy" } } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1, + controllerGeneration: 1, leaseOwner: `executor-${runId}`, leaseExpiresAt: new Date(Date.now() - 1_000) }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId, issueId, + status: "active", leasePolicy: "ephemeral", provider: "fake-plugin", providerLeaseId, + metadata: { ...config, agentId, driver: "sandbox", pluginId, pluginKey, sandboxProviderPlugin: true, + remoteCwd: "/workspace", sandboxState: "started" } }); + const events: Array<{ eventType: string; sourceEventId?: string }> = []; + if (!PRE_LAUNCH.includes(window)) events.push({ eventType: PROCESS_START_REQUESTED }); + if (window === "transport_connect" || window === "session_open") events.push({ eventType: PROCESS_IDENTITY_RECORDED }); + if (window === "session_open") events.push({ eventType: "session.started", sourceEventId: randomUUID() }); + for (const [index, event] of events.entries()) { + await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: index + 1, eventType: event.eventType, + stream: "system", level: "info", message: event.eventType, sourceEventId: event.sourceEventId ?? null }); + } + await db.update(heartbeatRuns).set({ nextEventSeq: events.length + 1 }).where(eq(heartbeatRuns.id, runId)); + return { companyId, agentId, issueId, runId, leaseId, providerLeaseId, ...providerWorker(pluginId) }; + } + + async function lease(leaseId: string) { + return (await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]!; + } + + async function run(runId: string) { + return (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!; + } + + async function expectSandboxReleased(fixture: Awaited>) { + let row = await lease(fixture.leaseId); + if (row.status === "pending_cleanup") { + await heartbeatService(db, { pluginWorkerManager: fixture.manager }).sweepPendingCleanupLeases({ backoffMs: 0 }); + row = await lease(fixture.leaseId); + } + expect(row.status).not.toBe("active"); + expect(row.status).not.toBe("pending_cleanup"); + expect(row.releasedAt).not.toBeNull(); + expect(fixture.teardownCalls()).toContainEqual([expect.any(String), expect.any(String), + expect.objectContaining({ providerLeaseId: fixture.providerLeaseId }), expect.any(Number)]); + } + + it.each(ALL_WINDOWS)("terminating the agent releases a sandbox leased during %s", async window => { + const fixture = await seed(window); + const lifecycle = createAgentLifecycle(db); + const worker = configureAgentLifecycle(db, createLifecycleDriver(db, fixture.manager)); + workers.push(worker); + await lifecycle.pauseAgent(fixture.agentId); + await worker.process(fixture.agentId); + await lifecycle.terminateAgent(fixture.agentId); + let agent = (await db.select().from(agents).where(eq(agents.id, fixture.agentId)))[0]!; + let passes = 0; + for (; agent.lifecycleState !== "terminated" && passes < 4; passes++) { + await db.update(agents).set({ lifecycleOperation: { ...agent.lifecycleOperation!, retryAt: undefined } }) + .where(eq(agents.id, fixture.agentId)); + await worker.process(fixture.agentId); + agent = (await db.select().from(agents).where(eq(agents.id, fixture.agentId)))[0]!; + } + expect(agent.lifecycleState).toBe("terminated"); + expect(passes).toBeLessThanOrEqual(1); + const stopped = await run(fixture.runId); + expect(stopped.status).toBe("cancelled"); + await expectSandboxReleased(fixture); + if (PRE_LAUNCH.includes(window)) { + expect(stopped.runnerProfileJson).not.toHaveProperty("nativeWorkspaceSync"); + expect(stopped.runnerProfileJson).toHaveProperty("nativeWorkspaceSyncReleasedUnchanged.state", "prepared"); + } else { + expect(stopped.resultJson).toHaveProperty("workspaceRestoreRecovery.schema", "paperclip.workspace-restore-recovery.v1"); + } + }, 60_000); + + it.each(ALL_WINDOWS)("stopping the run releases a sandbox leased during %s", async window => { + const fixture = await seed(window); + const heartbeat = heartbeatService(db, { pluginWorkerManager: fixture.manager }); + await heartbeat.cancelRun(fixture.runId, "Cancelled by a board operator", { + resultJson: { cancelledByActorType: "user", cancelledByUserId: "operator" } }); + expect((await run(fixture.runId)).status).toBe("cancelled"); + await heartbeat.sweepOrphanedActiveLeases({ companyId: fixture.companyId, leaseId: fixture.leaseId, backoffMs: 5 * 60_000 }); + await expectSandboxReleased(fixture); + }, 60_000); + + it("records a release a live workspace owner holds back and finishes it in the background once due", async () => { + const fixture = await seed("runner_launch"); + const ownerUntil = new Date(Date.now() + 60_000); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: ownerUntil }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "cancelled" }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const heartbeat = heartbeatService(db, { pluginWorkerManager: fixture.manager }); + await heartbeat.releaseEnvironmentLeasesForRun({ runId: fixture.runId, companyId: fixture.companyId, + agentId: fixture.agentId, status: "cancelled", providerResourceDisposition: "destroy" }); + const deferred = await lease(fixture.leaseId); + expect(deferred.status).toBe("active"); + expect(deferred.metadata?.releaseDeferral).toMatchObject({ reason: "workspace_owner_active", attempts: 1, + providerResourceDisposition: "destroy" }); + expect(fixture.teardownCalls()).toHaveLength(0); + + const deadlines: number[] = []; + await heartbeat.sweepOrphanedActiveLeases({ companyId: fixture.companyId, leaseId: fixture.leaseId, backoffMs: 5 * 60_000, + onDeferredDeadline: at => deadlines.push(at) }); + expect((await lease(fixture.leaseId)).status).toBe("active"); + expect(deadlines).toContain(Date.parse(String((deferred.metadata?.releaseDeferral as { retryAt: string }).retryAt))); + + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await db.update(environmentLeases).set({ metadata: { ...deferred.metadata, + releaseDeferral: { ...(deferred.metadata?.releaseDeferral as object), retryAt: new Date(Date.now() - 1_000).toISOString() } } }) + .where(eq(environmentLeases.id, fixture.leaseId)); + await heartbeat.sweepOrphanedActiveLeases({ companyId: fixture.companyId, leaseId: fixture.leaseId, backoffMs: 5 * 60_000 }); + await expectSandboxReleased(fixture); + }, 60_000); + + async function terminate(fixture: Awaited>) { + const lifecycle = createAgentLifecycle(db); + const worker = configureAgentLifecycle(db, createLifecycleDriver(db, fixture.manager)); + workers.push(worker); + await lifecycle.terminateAgent(fixture.agentId); + await worker.process(fixture.agentId); + return (await db.select().from(agents).where(eq(agents.id, fixture.agentId)))[0]!; + } + + it("finishes a release that lock conflicts held back, preserving the files, once the locks clear", async () => { + const fixture = await seed("runner_launch"); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "cancelled" }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const heartbeat = heartbeatService(db, { pluginWorkerManager: fixture.manager }); + let ready!: () => void, unlock!: () => void; + const locked = new Promise(resolve => { ready = resolve; }); + const released = new Promise(resolve => { unlock = resolve; }); + const holder = db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, fixture.issueId)).for("update"); + ready(); await released; + }); + await locked; + try { + for (let attempt = 1; attempt <= 3; attempt++) { + await heartbeat.releaseEnvironmentLeasesForRun({ runId: fixture.runId, companyId: fixture.companyId, + agentId: fixture.agentId, status: "cancelled", providerResourceDisposition: "destroy" }); + expect((await lease(fixture.leaseId)).metadata?.releaseDeferral).toMatchObject({ reason: "workspace_retention_busy", attempts: attempt }); + } + } finally { unlock(); await holder; } + const deferred = await lease(fixture.leaseId); + await db.update(environmentLeases).set({ metadata: { ...deferred.metadata, releaseDeferral: { + ...(deferred.metadata?.releaseDeferral as object), retryAt: new Date(Date.now() - 1_000).toISOString() } } }) + .where(eq(environmentLeases.id, fixture.leaseId)); + await heartbeat.sweepOrphanedActiveLeases({ companyId: fixture.companyId, leaseId: fixture.leaseId, backoffMs: 5 * 60_000 }); + await expectSandboxReleased(fixture); + expect((await run(fixture.runId)).resultJson).toHaveProperty("workspaceRestoreRecovery.schema", "paperclip.workspace-restore-recovery.v1"); + expect((await terminate(fixture)).lifecycleState).toBe("terminated"); + }, 60_000); + + it("attempts a slow-phase deferred release at once when the agent is terminated", async () => { + const fixture = await seed("runner_launch"); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "cancelled" }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const current = await lease(fixture.leaseId); + await db.update(environmentLeases).set({ metadata: { ...current.metadata, releaseDeferral: { reason: "release_failed", + attempts: 8, retryAt: new Date(Date.now() + 60 * 60_000).toISOString(), failingSince: new Date(Date.now() - 3 * 3_600_000).toISOString(), + providerResourceDisposition: "destroy" } } }).where(eq(environmentLeases.id, fixture.leaseId)); + expect((await terminate(fixture)).lifecycleState).toBe("terminated"); + await expectSandboxReleased(fixture); + }, 60_000); + + it("gives each overlapping stop operation one immediate attempt, then respects the stored retry time", async () => { + const fixture = await seed("runner_launch"); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60 * 60_000) }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "cancelled" }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const heartbeat = heartbeatService(db, { pluginWorkerManager: fixture.manager }); + const parent = `lifecycle:${randomUUID()}`, child = `lifecycle:${randomUUID()}`; + for (let pass = 0; pass < 30; pass++) { + expect(await heartbeat.agentInvocationStopBlocker([fixture.agentId], "Agent lifecycle stop requested", + { trigger: pass % 2 === 0 ? parent : child })).toMatch(/^environment_lease_release_deferred:workspace_owner_active$/); + } + expect((await lease(fixture.leaseId)).metadata?.releaseDeferral).toMatchObject({ reason: "workspace_owner_active", + attempts: 2, triggers: [parent, child] }); + expect(fixture.teardownCalls()).toHaveLength(0); + }, 60_000); + + it.each(PRE_LAUNCH)("the orphan lease check releases a sandbox a cancelled run leaked during %s", async window => { + const fixture = await seed(window); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "cancelled" }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const heartbeat = heartbeatService(db, { pluginWorkerManager: fixture.manager }); + await heartbeat.sweepOrphanedActiveLeases({ companyId: fixture.companyId, leaseId: fixture.leaseId, backoffMs: 5 * 60_000 }); + await expectSandboxReleased(fixture); + }, 60_000); +}); diff --git a/server/src/__tests__/heartbeat-terminate-pending-native-retry.test.ts b/server/src/__tests__/heartbeat-terminate-pending-native-retry.test.ts new file mode 100644 index 00000000000..1ab7ee3c733 --- /dev/null +++ b/server/src/__tests__/heartbeat-terminate-pending-native-retry.test.ts @@ -0,0 +1,140 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + companies, + createDb, + environmentLeases, + environments, + heartbeatRunEvents, + heartbeatRuns, + issues, + nativeRunFinalizations, + plugins, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); +vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); + +import { createLifecycleDriver } from "../services/agent-lifecycle-driver.js"; +import { configureAgentLifecycle, createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "../services/native-local-process-stop.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = support.supported ? describe : describe.skip; + +describeEmbeddedPostgres("terminating an agent whose failed run has a scheduled bootstrap retry", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + const workers: Array<{ stop(): Promise }> = []; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-terminate-pending-retry-"); + db = createDb(tempDb.connectionString); + }, 30_000); + + afterEach(async () => { + for (const worker of workers.splice(0)) await worker.stop(); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seed(issueStatus: "in_progress" | "cancelled") { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const environmentId = randomUUID(), leaseId = randomUUID(), pluginId = randomUUID(); + const providerLeaseId = `sandbox-${randomUUID()}`; + const pluginKey = `paperclip.pending-retry-${pluginId.slice(0, 8)}`; + await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", + apiVersion: 1, categories: ["automation"], status: "ready", installOrder: 1, updatedAt: new Date(), + manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Fake sandbox", description: "Test provider", + author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [{ driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", + configSchema: { type: "object" } }] } } as never); + await db.insert(companies).values({ id: companyId, name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "OpenCoder", role: "engineer", status: "idle", + adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" }, runtimeConfig: {}, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, title: "Short task", status: issueStatus, + priority: "medium", assigneeAgentId: agentId }); + const config = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false }; + await db.insert(environments).values({ id: environmentId, name: `Fake sandbox ${environmentId}`, driver: "sandbox", + status: "active", config }); + const failedAt = new Date(Date.now() - 5_000); + const retryAt = new Date(Date.now() + 17 * 60_000); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", status: "failed", + runtimeMode: "native", runtimeModeResolvedAt: failedAt, issueId, nativeIssueId: issueId, startedAt: failedAt, + finishedAt: failedAt, errorCode: "native_runner_process_exited", error: "The runner process exited", + contextSnapshot: { issueId }, nextEventSeq: 3, nativePhase: "retryable_failure", + resultJson: { executionRecovery: { kind: "bootstrap", recoverable: true, recoveryMode: "bootstrap_retry", + nextAttemptAt: retryAt.toISOString() } }, + runnerProfileJson: { nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId, providerLeaseId, + remoteCwd: "/workspace", resourceDisposition: "destroy" } } }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "retryable_failure", attempt: 1, + controllerGeneration: 1, failureCode: "native_runner_process_exited", recoveryState: "scheduled", nextAttemptAt: retryAt }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId, issueId, + status: "active", leasePolicy: "ephemeral", provider: "fake-plugin", providerLeaseId, + metadata: { ...config, agentId, driver: "sandbox", pluginId, pluginKey, sandboxProviderPlugin: true, + remoteCwd: "/workspace", sandboxState: "started" } }); + for (const [index, eventType] of [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED].entries()) { + await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: index + 1, eventType, + stream: "system", level: "info", message: eventType }); + } + const call = vi.fn(async (_pluginId: string, method: string, params: { providerLeaseId?: string }) => { + if (method === "environmentStopLease") return { providerLeaseId: params.providerLeaseId, state: "stopped" }; + if (method === "environmentDestroyLease" || method === "environmentReleaseLease") { + return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; + } + throw new Error(`Unexpected provider method ${method}`); + }); + const manager = { + isRunning: vi.fn((id: string) => id === pluginId), + getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", "environmentStopLease"] })), + call, + } as unknown as PluginWorkerManager; + return { companyId, agentId, issueId, runId, leaseId, providerLeaseId, manager, call }; + } + + it.each([ + ["in_progress", "in_progress"], + ["cancelled", "cancelled"], + ] as const)("cancels the pending retry and releases its sandbox at once (task %s)", async (issueStatus, finalStatus) => { + const fixture = await seed(issueStatus); + const lifecycle = createAgentLifecycle(db); + const worker = configureAgentLifecycle(db, createLifecycleDriver(db, fixture.manager)); + workers.push(worker); + const startedAt = Date.now(); + await lifecycle.pauseAgent(fixture.agentId); + await worker.process(fixture.agentId); + await lifecycle.terminateAgent(fixture.agentId); + await worker.process(fixture.agentId); + const agent = (await db.select().from(agents).where(eq(agents.id, fixture.agentId)))[0]!; + expect(agent.lifecycleState).toBe("terminated"); + expect(Date.now() - startedAt).toBeLessThan(15_000); + + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run!.status).toBe("cancelled"); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "terminal_failure", nextAttemptAt: null, leaseOwner: null }); + + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, fixture.leaseId)); + expect(lease!.status).not.toBe("active"); + expect(lease!.releasedAt).not.toBeNull(); + expect(fixture.call).toHaveBeenCalledWith(expect.any(String), expect.stringMatching(/^environment(Stop|Destroy|Release)Lease$/), + expect.objectContaining({ providerLeaseId: fixture.providerLeaseId }), expect.any(Number)); + + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, fixture.companyId), + eq(heartbeatRuns.retryOfRunId, fixture.runId)))).toHaveLength(0); + const [task] = await db.select().from(issues).where(eq(issues.id, fixture.issueId)); + expect(task!.status).toBe(finalStatus); + expect(run!.resultJson).toMatchObject({ pauseStop: { holds: ["manual"] }, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [fixture.leaseId] } }); + }, 60_000); +}); diff --git a/server/src/modules/agent-lifecycle/__tests__/driver-scope.test.ts b/server/src/modules/agent-lifecycle/__tests__/driver-scope.test.ts index da514753201..96d31c984af 100644 --- a/server/src/modules/agent-lifecycle/__tests__/driver-scope.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/driver-scope.test.ts @@ -1,8 +1,8 @@ import type { Db } from "@paperclipai/db"; import { describe, expect, it, vi } from "vitest"; -const stop = vi.hoisted(() => vi.fn(async () => true)); -vi.mock("../../../services/heartbeat.js", () => ({ heartbeatService: () => ({ stopInvocationsForAgents: stop }) })); +const stop = vi.hoisted(() => vi.fn(async () => null)); +vi.mock("../../../services/heartbeat.js", () => ({ heartbeatService: () => ({ agentInvocationStopBlocker: stop }) })); vi.mock("../../../services/agent-harness-verification.js", () => ({ agentHarnessVerificationService: () => ({ verify: vi.fn() }) })); import { createLifecycleDriver } from "../../../services/agent-lifecycle-driver.js"; diff --git a/server/src/modules/agent-lifecycle/__tests__/pending-backoff.test.ts b/server/src/modules/agent-lifecycle/__tests__/pending-backoff.test.ts new file mode 100644 index 00000000000..ea7da2eae97 --- /dev/null +++ b/server/src/modules/agent-lifecycle/__tests__/pending-backoff.test.ts @@ -0,0 +1,86 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { configureAgentLifecycle, createAgentLifecycle } from "../../../services/agent-lifecycle.js"; +import { lifecyclePendingRetryDelayMs } from "../adapters/postgres.js"; +import { wakeStoppingAgentLifecycle } from "../../../services/environment-lease-release-deferral.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("lifecycle host step that stays pending", () => { + let database: Awaited> | undefined; + let db: Db; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-lifecycle-pending-backoff-"); + db = createDb(database.connectionString); + }, 60_000); + + afterAll(async () => { + await database?.cleanup(); + }); + + it("records why termination is waiting and spaces out its passes", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Lifecycle", issuePrefix: `P${companyId.slice(0, 7)}` }); + const [agent] = await db.insert(agents).values({ companyId, name: "Stuck", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: {}, runtimeConfig: {}, permissions: {} }).returning(); + let hostCalls = 0; + const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", + runHost: async () => { hostCalls += 1; return { pending: "environment_lease_unreleased" }; } }); + try { + await createAgentLifecycle(db).terminateAgent(agent!.id); + const delays: number[] = []; + for (let pass = 1; pass <= 7; pass++) { + const before = Date.now(); + await worker.process(agent!.id); + await worker.process(agent!.id); + const [current] = await db.select().from(agents).where(eq(agents.id, agent!.id)); + expect(current!.lifecycleState).toBe("terminating"); + expect(current!.lifecycleOperation).toMatchObject({ pendingReason: "environment_lease_unreleased", pendingPasses: pass }); + delays.push(Date.parse(current!.lifecycleOperation!.retryAt!) - before); + await db.update(agents).set({ lifecycleOperation: { ...current!.lifecycleOperation!, retryAt: new Date(0).toISOString() } }) + .where(eq(agents.id, agent!.id)); + } + expect(hostCalls).toBe(7); + for (const [index, delay] of delays.entries()) { + expect(delay).toBeGreaterThanOrEqual(lifecyclePendingRetryDelayMs(index + 1) - 1_000); + } + expect(delays.at(-1)).toBeGreaterThanOrEqual(60_000 - 1_000); + expect(delays.at(-1)).toBeLessThanOrEqual(60_000 + 1_000); + expect(lifecyclePendingRetryDelayMs(11)).toBe(60_000); + expect(lifecyclePendingRetryDelayMs(500)).toBe(60_000); + } finally { + await worker.stop(); + } + }, 60_000); + + it("finishes a stop at once when its blocker clears, without waiting out the backoff", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Lifecycle", issuePrefix: `W${companyId.slice(0, 7)}` }); + const [agent] = await db.insert(agents).values({ companyId, name: "Waiting", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: {}, runtimeConfig: {}, permissions: {} }).returning(); + let blocked = true; + const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", + runHost: async () => blocked ? { pending: "environment_lease_unreleased" } : "complete" }); + try { + await createAgentLifecycle(db).terminateAgent(agent!.id); + for (let pass = 1; pass <= 11; pass++) { + await worker.process(agent!.id); + const [current] = await db.select().from(agents).where(eq(agents.id, agent!.id)); + if (pass < 11) await db.update(agents).set({ lifecycleOperation: { ...current!.lifecycleOperation!, + retryAt: new Date(0).toISOString() } }).where(eq(agents.id, agent!.id)); + } + const [waiting] = await db.select().from(agents).where(eq(agents.id, agent!.id)); + expect(Date.parse(waiting!.lifecycleOperation!.retryAt!)).toBeGreaterThan(Date.now() + 30_000); + blocked = false; + expect(await db.transaction(tx => wakeStoppingAgentLifecycle(tx as unknown as Db, agent!.id))).toBe(true); + await worker.process(agent!.id); + expect((await db.select().from(agents).where(eq(agents.id, agent!.id)))[0]!.lifecycleState).toBe("terminated"); + } finally { + await worker.stop(); + } + }, 60_000); +}); diff --git a/server/src/modules/agent-lifecycle/adapters/postgres.ts b/server/src/modules/agent-lifecycle/adapters/postgres.ts index 6378e51dd05..99871678dc5 100644 --- a/server/src/modules/agent-lifecycle/adapters/postgres.ts +++ b/server/src/modules/agent-lifecycle/adapters/postgres.ts @@ -11,6 +11,13 @@ export function assertRootDatabase(db: Db) { } type LifecycleRow = typeof agents.$inferSelect; + +const LIFECYCLE_PENDING_BASE_MS = 2_000; +const LIFECYCLE_PENDING_MAX_MS = 60_000; + +export function lifecyclePendingRetryDelayMs(passes: number) { + return Math.min(LIFECYCLE_PENDING_MAX_MS, LIFECYCLE_PENDING_BASE_MS * 2 ** Math.max(0, passes - 1)); +} type ChangeOptions = { version?: number; owner?: string; reason?: string }; // Prepared drafts wait for the wizard; they have no background verification obligation. @@ -83,6 +90,24 @@ export async function requestAgentLifecycleRecheckInTransaction( return true; } +export async function wakeStoppingAgentLifecycleInTransaction( + tx: Pick, + agentId: string, + notify: () => Promise, +): Promise { + if (!("nestedIndex" in tx)) throw new AgentLifecycleConflict("Lifecycle wakes require the producer transaction"); + const [agent] = await tx.select({ lifecycleState: agents.lifecycleState, lifecycleOperation: agents.lifecycleOperation }) + .from(agents).where(eq(agents.id, agentId)).for("update"); + const operation = agent?.lifecycleOperation; + if (!agent || !operation || !["pausing", "terminating", "cleaning_up"].includes(agent.lifecycleState) + || (!operation.retryAt && !operation.leaseOwner)) return false; + await notify(); + await tx.update(agents).set({ lifecycleOperation: sql`(${agents.lifecycleOperation} - 'retryAt') + || case when ${agents.lifecycleOperation} ? 'leaseOwner' then '{"recheckRequested":true}'::jsonb else '{}'::jsonb end` }) + .where(eq(agents.id, agentId)); + return true; +} + /** The caller owns the accounting transaction and has locked this agent. */ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: LifecycleRow, command: LifecycleCommand, options: ChangeOptions = {}, policyHolds?: string[], restartPause = false) { @@ -268,9 +293,12 @@ export function createLifecycleStore(db: Db, effects: LifecycleEffects): Lifecyc sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); return rows.length > 0; } + const previousPasses = (agent.lifecycleOperation as { pendingPasses?: unknown } | null)?.pendingPasses; + const pendingPasses = options?.pendingReason ? (typeof previousPasses === "number" ? previousPasses : 0) + 1 : undefined; const retry: Partial = options?.retryRequired && ["verifying", "resuming"].includes(agent.lifecycleState) ? { retryRequired: true } - : { retryAt: new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString() }; + : { retryAt: new Date(now.getTime() + (error ? 60_000 : pendingPasses ? lifecyclePendingRetryDelayMs(pendingPasses) : 2_000)).toISOString() }; + if (options?.pendingReason && pendingPasses) Object.assign(retry, { pendingReason: options.pendingReason, pendingPasses }); if (options?.verificationFailures !== undefined && agent.lifecycleState === "verifying") { retry.verificationFailures = options.verificationFailures; } diff --git a/server/src/modules/agent-lifecycle/application/ports.ts b/server/src/modules/agent-lifecycle/application/ports.ts index ade8e539011..a7e99599c30 100644 --- a/server/src/modules/agent-lifecycle/application/ports.ts +++ b/server/src/modules/agent-lifecycle/application/ports.ts @@ -20,14 +20,14 @@ export interface LifecycleStore { setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise; completeHost(agent: LifecycleAgent, owner: string): Promise; completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise; - defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date, options?: { retryRequired?: boolean; verificationFailures?: number; waitForSignal?: boolean }): Promise; + defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date, options?: { retryRequired?: boolean; verificationFailures?: number; waitForSignal?: boolean; pendingReason?: string }): Promise; pending(limit: number, now: Date): Promise; } export interface LifecycleDriver { requiredPluginIds(agent: LifecycleAgent): Promise; /** waiting is only a healthy verifying phase owned by an external event. */ - runHost(agent: LifecycleAgent): Promise<"complete" | "pending" | "waiting">; + runHost(agent: LifecycleAgent): Promise<"complete" | "pending" | "waiting" | { pending: string }>; runPlugin(agent: LifecycleAgent, pluginId: string): Promise<"complete" | "pending">; /** Notify the scheduler only after readiness has committed. */ onReady?(agent: LifecycleAgent): Promise; diff --git a/server/src/modules/agent-lifecycle/application/worker.ts b/server/src/modules/agent-lifecycle/application/worker.ts index 5a83aec531e..b45f0564533 100644 --- a/server/src/modules/agent-lifecycle/application/worker.ts +++ b/server/src/modules/agent-lifecycle/application/worker.ts @@ -51,8 +51,9 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr if (!await runStep("defer", () => store.defer(agent, owner, null, new Date(), { waitForSignal: true }))) continue; return; } - if (host === "pending") { - if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue; + if (host === "pending" || typeof host === "object") { + const pendingReason = typeof host === "object" ? host.pending : "host_step_pending"; + if (!await runStep("defer", () => store.defer(agent, owner, null, new Date(), { pendingReason }))) continue; return; } if (!await runStep("complete_host", () => store.completeHost(agent, owner))) continue; @@ -65,7 +66,7 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr } if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue phase; if (await runStep("plugin", () => driver.runPlugin(agent, pluginId), pluginId) === "pending") { - if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue phase; + if (!await runStep("defer", () => store.defer(agent, owner, null, new Date(), { pendingReason: "plugin_step_pending" }))) continue phase; return; } if (!await runStep("complete_plugin", () => store.completePlugin(agent, owner, pluginId), pluginId)) continue phase; diff --git a/server/src/modules/agent-lifecycle/index.ts b/server/src/modules/agent-lifecycle/index.ts index a958e8e65b9..33c9d8f73ae 100644 --- a/server/src/modules/agent-lifecycle/index.ts +++ b/server/src/modules/agent-lifecycle/index.ts @@ -15,7 +15,7 @@ import { hasCompanyHoldMismatch, projectCompanyHolds } from "./adapters/company- import { createLifecycleWorker } from "./application/worker.js"; import type { LifecycleDriver } from "./application/ports.js"; -export { applyAgentPolicyHoldsInTransaction, releaseAgentLifecycleLeaseOfHolderInTransaction, requestAgentLifecycleRecheckInTransaction } from "./adapters/postgres.js"; +export { applyAgentPolicyHoldsInTransaction, releaseAgentLifecycleLeaseOfHolderInTransaction, requestAgentLifecycleRecheckInTransaction, wakeStoppingAgentLifecycleInTransaction } from "./adapters/postgres.js"; export { companyHoldMismatch, fenceActiveCompanyAgentStopsInTransaction } from "./adapters/company-policy.js"; export { AgentLifecycleConflict, canConfigureAgentConnection, isAgentAwaitingSetup, isAgentSetupDraft } from "./domain/policy.js"; diff --git a/server/src/services/agent-execution-stop.ts b/server/src/services/agent-execution-stop.ts index 92e9e5964bb..481b216e6d3 100644 --- a/server/src/services/agent-execution-stop.ts +++ b/server/src/services/agent-execution-stop.ts @@ -1,4 +1,5 @@ import { readProcessStartedAt } from "./hot-restart.js"; +import { readLeaseReleaseDeferral } from "./environment-lease-release-deferral.js"; import { runUsedConversationAdapter } from "./conversation-continuation.js"; import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; @@ -8,20 +9,26 @@ import { isCancelledNativeStartup } from "./cancelled-native-startup.js"; /** Cancellation changes admission. Process and lease evidence controls cleanup. */ export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { - if (!agentIds.length) return true; + return (await agentExecutionStopBlocker(db, agentIds)) === null; +} + +export async function agentExecutionStopBlocker(db: Db, agentIds: string[]): Promise { + if (!agentIds.length) return null; const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); const runIds = db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); const leases = await db.select().from(environmentLeases).where(or( inArray(environmentLeases.heartbeatRunId, runIds), inArray(sql`${environmentLeases.metadata}->>'agentId'`, agentIds), )); - if (leases.some(lease => (!lease.releasedAt && !(lease.status === "retained" && lease.cleanupStatus === "success")) || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed")) return false; + const deferred = leases.find(lease => lease.status === "active" && readLeaseReleaseDeferral(lease.metadata)); + if (deferred) return `environment_lease_release_deferred:${readLeaseReleaseDeferral(deferred.metadata)!.reason}`; + if (leases.some(lease => (!lease.releasedAt && !(lease.status === "retained" && lease.cleanupStatus === "success")) || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed")) return "environment_lease_unreleased"; const coordinators = await db.select().from(nativeRunFinalizations) .where(inArray(nativeRunFinalizations.runId, runIds)); const now = new Date(); const abandoned = coordinators.filter(row => row.leaseOwner); if (abandoned.some(row => !row.leaseExpiresAt || row.leaseExpiresAt > now || row.resultId || - runs.find(run => run.id === row.runId)?.status !== "cancelled")) return false; + runs.find(run => run.id === row.runId)?.status !== "cancelled")) return "native_finalization_owned"; const absent = (pid: number) => { try { process.kill(pid, 0); return false; } catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; } @@ -30,14 +37,14 @@ export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { if (!coordinator.controllerPid || absent(coordinator.controllerPid)) continue; const observed = await readProcessStartedAt(coordinator.controllerPid).catch(() => null); if (!observed || !coordinator.controllerProcessStartedAt || - new Date(observed).getTime() === coordinator.controllerProcessStartedAt.getTime()) return false; + new Date(observed).getTime() === coordinator.controllerProcessStartedAt.getTime()) return "native_controller_alive"; } for (const run of runs) { - if (!["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status)) return false; - if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) return false; + if (!["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status)) return "run_not_terminal"; + if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) return "run_controller_lease_held"; const runLeases = leases.filter(lease => lease.heartbeatRunId === run.id); if (runLeases.some(lease => lease.provider && lease.provider !== "local")) { - if (run.status === "cancelled" && !runLeases.every(hasRemoteTerminationReceipt)) return false; + if (run.status === "cancelled" && !runLeases.every(hasRemoteTerminationReceipt)) return "remote_stop_unconfirmed"; continue; } let processAlive = Boolean(run.processPid && !absent(run.processPid)); @@ -49,15 +56,15 @@ export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { if (run.processGroupId === run.processPid) groupAlive = false; } } - if (processAlive || groupAlive) return false; + if (processAlive || groupAlive) return "run_process_alive"; if (run.status !== "cancelled" || !run.startedAt) continue; if (run.runtimeMode === "native" && !run.processPid && !run.processGroupId && !await hasNativeLocalProcessStop(db, run.companyId, run.id) && - !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false; + !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return "native_stop_unproven"; if (run.runtimeMode === "legacy" && !run.processPid && !run.processGroupId && (run.resultJson?.executionCancellation as Record | undefined)?.state !== "acknowledged" && !(run.executionStage === "settled" && !await runUsedConversationAdapter(db, run)) && - !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false; + !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return "legacy_stop_unproven"; } // Release an expired owner only after its cancelled execution has stop proof. for (const coordinator of abandoned) { @@ -68,7 +75,7 @@ export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { eq(nativeRunFinalizations.controllerGeneration, coordinator.controllerGeneration), eq(nativeRunFinalizations.phase, coordinator.phase), isNull(nativeRunFinalizations.resultId))) .returning({ runId: nativeRunFinalizations.runId }); - if (!cleared) return false; + if (!cleared) return "native_finalization_release_raced"; } - return true; + return null; } diff --git a/server/src/services/agent-lifecycle-driver.ts b/server/src/services/agent-lifecycle-driver.ts index a74ff0c5115..8cbc3a2c272 100644 --- a/server/src/services/agent-lifecycle-driver.ts +++ b/server/src/services/agent-lifecycle-driver.ts @@ -1,4 +1,5 @@ -import { and, eq, inArray, sql } from "drizzle-orm"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; +import { pendingFailedNativeRetryCancellationCondition } from "./native-runtime/native-cancellation-request.js"; import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./heartbeat/recovery.js"; import { promoteDeferredWorkIntents } from "./work-intent-outbox.js"; import { agents, companies, environments, heartbeatRuns, plugins, pluginCompanySettings, type Db } from "@paperclipai/db"; @@ -87,11 +88,13 @@ export function createLifecycleDriver(db: Db, manager: PluginWorkerManager): Lif await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${pauseStop}`, }).where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + or(inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), pendingFailedNativeRetryCancellationCondition()), sql`not (coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ? 'pauseStop')`)); } - if (!await heartbeat.stopInvocationsForAgents(ids, reason, - { preservePauseDeferred: agent.lifecycleState === "pausing" })) return "pending"; + const blocker = await heartbeat.agentInvocationStopBlocker(ids, reason, + { preservePauseDeferred: agent.lifecycleState === "pausing", + ...(agent.lifecycleOperation?.id ? { trigger: `lifecycle:${agent.lifecycleOperation.id}` } : {}) }); + if (blocker) return { pending: blocker }; } return "complete"; }, diff --git a/server/src/services/environment-lease-release-deferral.test.ts b/server/src/services/environment-lease-release-deferral.test.ts new file mode 100644 index 00000000000..1b867624f1b --- /dev/null +++ b/server/src/services/environment-lease-release-deferral.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; + +const boot = vi.hoisted(() => ({ id: "boot-a" })); +vi.mock("./controller-process-identity.js", async importOriginal => ({ + ...await importOriginal(), + currentControllerBootId: () => boot.id, +})); + +import { leaseReleaseRetryDueAt, nextLeaseReleaseBootIds, nextLeaseReleaseRetryAt, + type LeaseReleaseDeferral } from "./environment-lease-release-deferral.js"; + +describe("lease release retry across live server boots", () => { + function fail(previous: LeaseReleaseDeferral | null, now: number): LeaseReleaseDeferral { + const attempts = (previous?.attempts ?? 0) + 1; + return { reason: "release_failed", attempts, retryAt: nextLeaseReleaseRetryAt(previous, attempts, now), + failingSince: previous?.failingSince ?? new Date(now).toISOString(), attemptedByBootIds: nextLeaseReleaseBootIds(previous), + providerResourceDisposition: null }; + } + + function simulate(boots: string[]) { + const start = Date.parse("2026-11-15T12:00:00Z"); + boot.id = boots[0]!; + let deferral = fail(null, start); + let attempts = 1; + let latestRetryAt = Date.parse(deferral.retryAt); + for (let minute = 1; minute <= 60; minute++) { + const now = start + minute * 60_000; + for (const id of [...boots].reverse()) { + boot.id = id; + if (leaseReleaseRetryDueAt(deferral, now) > now) continue; + deferral = fail(deferral, now); + attempts += 1; + const retryAt = Date.parse(deferral.retryAt); + expect(retryAt).toBeGreaterThanOrEqual(latestRetryAt); + latestRetryAt = retryAt; + } + } + return { deferral, attempts }; + } + + it("keeps the saved retry time for every live boot after each has made its first attempt", () => { + const single = simulate(["boot-a"]); + const overlapping = simulate(["boot-a", "boot-b"]); + expect(overlapping.deferral.attemptedByBootIds).toEqual(["boot-a", "boot-b"]); + expect(overlapping.attempts).toBeLessThanOrEqual(single.attempts + 1); + }); + + it("gives a restarted server one immediate attempt and keeps the boot list bounded", () => { + const now = Date.parse("2026-11-15T12:00:00Z"); + let deferral: LeaseReleaseDeferral | null = null; + for (let index = 0; index < 15; index++) { + boot.id = `boot-${index}`; + deferral = fail(deferral, now); + } + expect(deferral!.attemptedByBootIds).toHaveLength(10); + boot.id = "boot-restarted"; + expect(leaseReleaseRetryDueAt(deferral!, now)).toBe(now); + boot.id = "boot-14"; + expect(leaseReleaseRetryDueAt(deferral!, now)).toBe(Date.parse(deferral!.retryAt)); + }); +}); diff --git a/server/src/services/environment-lease-release-deferral.ts b/server/src/services/environment-lease-release-deferral.ts new file mode 100644 index 00000000000..058b4510c89 --- /dev/null +++ b/server/src/services/environment-lease-release-deferral.ts @@ -0,0 +1,96 @@ +import { and, eq, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { currentControllerBootId } from "./controller-process-identity.js"; +import { wakeStoppingAgentLifecycleInTransaction } from "../modules/agent-lifecycle/index.js"; +import type { ProviderResourceDisposition } from "./environment-runtime.js"; +import { executionStartedEventPredicate } from "./pre-dispatch-execution-evidence.js"; + +export const LEASE_RELEASE_DEFERRAL_KEY = "releaseDeferral"; +const LEASE_RELEASE_RETRY_BASE_MS = 30_000; +const LEASE_RELEASE_RETRY_MAX_MS = 15 * 60_000; +export const LEASE_RELEASE_FAST_ATTEMPTS = 6; +export const LEASE_RELEASE_SLOW_RETRY_MS = 60 * 60_000; + +export type LeaseReleaseDeferral = { + reason: string; + attempts: number; + retryAt: string; + failingSince: string; + triggers?: string[]; + attemptedByBootIds?: string[]; + providerResourceDisposition: ProviderResourceDisposition | null; +}; + +const LEASE_RELEASE_TRIGGER_LIMIT = 20; + +const LEASE_RELEASE_BOOT_LIMIT = 10; + +export function leaseReleaseAttemptedByThisBoot(deferral: LeaseReleaseDeferral | null) { + return Boolean(deferral?.attemptedByBootIds?.includes(currentControllerBootId())); +} + +export function leaseReleaseRetryDueAt(deferral: LeaseReleaseDeferral, now = Date.now()) { + if (!leaseReleaseAttemptedByThisBoot(deferral)) return now; + const retryAt = Date.parse(deferral.retryAt); + return Number.isFinite(retryAt) ? retryAt : now; +} + +export function nextLeaseReleaseBootIds(previous: LeaseReleaseDeferral | null) { + const current = currentControllerBootId(); + const bootIds = (previous?.attemptedByBootIds ?? []).filter(bootId => bootId !== current); + return [...bootIds, current].slice(-LEASE_RELEASE_BOOT_LIMIT); +} + +export function nextLeaseReleaseRetryAt(previous: LeaseReleaseDeferral | null, attempts: number, now = Date.now()) { + const scheduled = now + leaseReleaseRetryDelayMs(attempts); + const saved = previous ? Date.parse(previous.retryAt) : Number.NaN; + return new Date(Number.isFinite(saved) && saved > scheduled ? saved : scheduled).toISOString(); +} + +export function nextLeaseReleaseTriggers(previous: LeaseReleaseDeferral | null, trigger: string | undefined) { + const triggers = [...(previous?.triggers ?? [])]; + if (trigger && !triggers.includes(trigger)) triggers.push(trigger); + return triggers.slice(-LEASE_RELEASE_TRIGGER_LIMIT); +} + +export function readLeaseReleaseDeferral(metadata: Record | null | undefined): LeaseReleaseDeferral | null { + const value = metadata?.[LEASE_RELEASE_DEFERRAL_KEY]; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const deferral = value as Record; + if (typeof deferral.reason !== "string" || typeof deferral.retryAt !== "string" || typeof deferral.attempts !== "number") return null; + return deferral as LeaseReleaseDeferral; +} + +export function leaseReleaseRetryDelayMs(attempts: number) { + if (attempts > LEASE_RELEASE_FAST_ATTEMPTS) return LEASE_RELEASE_SLOW_RETRY_MS; + return Math.min(LEASE_RELEASE_RETRY_MAX_MS, LEASE_RELEASE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1)); +} + +export function wakeStoppingAgentLifecycle(tx: Db, agentId: string) { + return wakeStoppingAgentLifecycleInTransaction(tx, agentId, () => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle)); +} + +export async function settleUnchangedNativeWorkspace(db: Db, runId: string, companyId: string): Promise { + const settled = await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`(${heartbeatRuns.runnerProfileJson} - 'nativeWorkspaceSync') + || jsonb_build_object('nativeWorkspaceSyncReleasedUnchanged', ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync')`, + }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "cancelled"), + sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'`, + sql`coalesce(${heartbeatRuns.runnerProfileJson}->'remoteWorkspace', 'null'::jsonb) = 'null'::jsonb`, + sql`not exists (select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${companyId} + and ${environmentLeases.heartbeatRunId} = ${runId} and (${environmentLeases.status} in ('active', 'pending_cleanup', 'retained') + or coalesce(${environmentLeases.metadata}, '{}'::jsonb) ? 'sandboxStopAndRetainReceipt'))`)) + .returning({ id: heartbeatRuns.id }); + return settled.length > 0; +} + +export async function nativeRunNeverLaunchedRunner(db: Db, runId: string, companyId: string): Promise { + const [run] = await db.select({ processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))); + if (!run || run.processPid || run.processGroupId) return false; + const [started] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, companyId), eq(heartbeatRunEvents.runId, runId), executionStartedEventPredicate(), + )).limit(1); + return !started; +} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 4d66ec03d4a..3648d2c5770 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -81,7 +81,7 @@ import { formatIssueIdentifierLink, } from "./heartbeat/scheduling.js"; import { canApplyTaskWorkspaceSelectionAtAdmission, taskWorkspaceRuntimeSelectionEnabled } from "./execution-workspace-policy.js"; -import { agentExecutionsHaveStopped } from "./agent-execution-stop.js"; +import { agentExecutionStopBlocker } from "./agent-execution-stop.js"; import { cancelHeartbeatNativeRun, terminateHeartbeatRunProcess, @@ -4444,31 +4444,45 @@ export function heartbeatService( } } - async function stopInvocationsForAgents(agentIds: string[], reason: string, options: StopInvocationOptions = {}) { + async function agentInvocationStopBlocker(agentIds: string[], reason: string, options: StopInvocationOptions = {}) { await cancelInvocationsForAgentsInternal(agentIds, reason, options); const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); const leasesToRelease = await db.select({ runId: environmentLeases.heartbeatRunId }).from(environmentLeases) .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, environmentLeases.heartbeatRunId)) .where(and(inArray(heartbeatRuns.agentId, agentIds), or(and(eq(heartbeatRuns.status, "cancelled"), inArray(environmentLeases.status, ["active", "pending_cleanup"])), - and(eq(environmentLeases.status, "retained"), eq(environmentLeases.cleanupStatus, "failed"))))); + and(eq(environmentLeases.status, "retained"), eq(environmentLeases.cleanupStatus, "failed"))), + sql`not (coalesce((${environmentLeases.metadata}->'releaseDeferral'->>'retryAt')::timestamptz > now(), false) + and (${options.trigger ?? null}::text is null + or coalesce(${environmentLeases.metadata}->'releaseDeferral'->'triggers', '[]'::jsonb) ? ${options.trigger ?? ""}::text))`)); const needsRelease = new Set(leasesToRelease.map(lease => lease.runId)); + let blocker: string | null = null; for (const run of runs) { - if (!isHeartbeatRunTerminalStatus(run.status) || liveRunExecutions.has(run.id) || - adapterExecutionControls.has(run.id) || processRunCancellationSettlements.has(run.id) || - (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date())) return false; + const runBlocker = !isHeartbeatRunTerminalStatus(run.status) ? "run_not_terminal" + : liveRunExecutions.has(run.id) || adapterExecutionControls.has(run.id) || processRunCancellationSettlements.has(run.id) + ? "run_executing_here" + : run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() ? "run_controller_lease_held" : null; + if (runBlocker) { + blocker ??= runBlocker; + continue; + } if (needsRelease.has(run.id)) { - // Retry incomplete cleanup after a run becomes terminal. await releaseEnvironmentLeasesForRun({ runId: run.id, companyId: run.companyId, - agentId: run.agentId, status: run.status, providerResourceDisposition: "destroy" }); + agentId: run.agentId, status: run.status, providerResourceDisposition: "destroy", + ...(options.trigger ? { trigger: options.trigger } : {}) }); } } - return agentExecutionsHaveStopped(db, agentIds); + return blocker ?? agentExecutionStopBlocker(db, agentIds); + } + + async function stopInvocationsForAgents(agentIds: string[], reason: string, options: StopInvocationOptions = {}) { + return (await agentInvocationStopBlocker(agentIds, reason, options)) === null; } return { recoverLegacyAccountingReceipt, stopInvocationsForAgents, + agentInvocationStopBlocker, waitForRunExecutionDrain: async ( runId: string, options: { timeoutMs?: number; intervalMs?: number } = {}, diff --git a/server/src/services/heartbeat/live-recovery-work.ts b/server/src/services/heartbeat/live-recovery-work.ts index 07bc8d14950..c5698decb97 100644 --- a/server/src/services/heartbeat/live-recovery-work.ts +++ b/server/src/services/heartbeat/live-recovery-work.ts @@ -7,6 +7,12 @@ import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; import type { LiveResourceRecoveryScope, LiveRunRecoveryScope } from "../live-run-recovery-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; +import { leaseReleaseRetryDueAt, readLeaseReleaseDeferral } from "../environment-lease-release-deferral.js"; + +function leaseDueAt(metadata: Record | null) { + const deferral = readLeaseReleaseDeferral(metadata); + return deferral ? leaseReleaseRetryDueAt(deferral) : 0; +} const TERMINAL = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"]; const SETTLED_PENDING_BASE_DELAY_MS = 30_000; @@ -64,7 +70,8 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType)?.deadlineAt)) || Date.now(), })); } else if (kind === "settled") { diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 186f855e7e7..1d595baf587 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -2,6 +2,7 @@ import { settleOwnableKind } from "../recovery/ownership-invariant-report.js"; import { isRunHandedOffForShutdown, markRunHandedOffForShutdown } from "./shutdown-handoff.js"; import { ORPHANED_RUN_LEASE_FAILURE_REASON } from "../environment-run-lease-settlement.js"; import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "../environment-lease-controller.js"; +import { leaseReleaseRetryDueAt, readLeaseReleaseDeferral } from "../environment-lease-release-deferral.js"; import { readNativeStartupCleanup, readNativeStartupContext } from "../native-runtime/native-startup-cleanup.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js"; @@ -1676,6 +1677,40 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD ); } + async function retryTerminalRunLeaseRelease(lease: typeof environmentLeases.$inferSelect, + localRuns: { has(runId: string): boolean }, onDeferredDeadline?: (at: number) => void): Promise { + if (!lease.heartbeatRunId || localRuns.has(lease.heartbeatRunId)) return false; + const deferral = readLeaseReleaseDeferral(lease.metadata); + const dueAt = deferral ? leaseReleaseRetryDueAt(deferral) : Number.NaN; + if (deferral && dueAt > Date.now()) { + onDeferredDeadline?.(dueAt); + return false; + } + const [run] = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, error: heartbeatRuns.error, runtimeMode: heartbeatRuns.runtimeMode, + controllerLeaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, lease.heartbeatRunId), eq(heartbeatRuns.companyId, lease.companyId))); + if (!run || !HEARTBEAT_RUN_TERMINAL_STATUSES.some(status => status === run.status)) return false; + if (!deferral && !(run.runtimeMode === "native" && run.status === "cancelled")) return false; + const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }) + .from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId))); + const heldUntil = Math.max(run.controllerLeaseExpiresAt?.getTime() ?? 0, + owner?.leaseOwner ? owner.leaseExpiresAt?.getTime() ?? 0 : 0); + if (heldUntil > Date.now()) { + onDeferredDeadline?.(heldUntil); + return false; + } + await releaseEnvironmentLeasesForRun({ runId: run.id, companyId: run.companyId, agentId: run.agentId, + status: run.status, failureReason: run.error ?? undefined, + providerResourceDisposition: deferral ? deferral.providerResourceDisposition ?? undefined : "destroy" }); + const [after] = await db.select({ status: environmentLeases.status, metadata: environmentLeases.metadata }) + .from(environmentLeases).where(eq(environmentLeases.id, lease.id)); + if (after?.status !== "active") return true; + const next = readLeaseReleaseDeferral(after.metadata); + if (next) onDeferredDeadline?.(Date.parse(next.retryAt)); + return false; + } + // A run's active lease outlives a crash between the terminal run write and // the lease release, or a release that skipped the lease (for example when // its environment row is gone). No production query finds such a lease by @@ -1730,6 +1765,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD && await nativeWorkspaceExportPendingForLease(db, lease) && await preserveOrphanedNativeWorkspace(db, lease, { onDeferredDeadline: opts.onDeferredDeadline })) recovered += 1; } + for (const { lease } of scoped) { + if (await retryTerminalRunLeaseRelease(lease, localRuns, opts.onDeferredDeadline)) recovered += 1; + } const runIds = [...new Set(scoped.flatMap(({ lease }) => lease.heartbeatRunId ? [lease.heartbeatRunId] : []))]; if (runIds.length > 0) { diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index 81955fcd065..bfbaff21d1f 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -30,6 +30,8 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { acknowledgedNativeApiStopExecutionHasStopped } from "../acknowledged-native-stop.js"; import { nativeSha256 } from "../native-runtime/canonical.js"; import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js"; +import { LEASE_RELEASE_FAST_ATTEMPTS, LEASE_RELEASE_SLOW_RETRY_MS } from "../environment-lease-release-deferral.js"; +import { agentExecutionStopBlocker } from "../agent-execution-stop.js"; vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); vi.mock("../acknowledged-native-stop.js", async (importOriginal) => { @@ -125,7 +127,7 @@ describe("heartbeat run-control module boundary", () => { expect(database.access).not.toHaveBeenCalled(); }); - it("defers destruction when a warm session remains busy", async () => { + it("defers destruction and records a retry when a warm session remains busy", async () => { const database = guardedDatabase(); const deps = callbacks(database.db); deps.getRunCleanupState.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); @@ -133,7 +135,7 @@ describe("heartbeat run-control module boundary", () => { await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" }); expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); - expect(database.access).not.toHaveBeenCalled(); + expect(database.access).toHaveBeenCalledWith(expect.anything(), "select", expect.anything()); }); }); @@ -869,6 +871,8 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, heartbeatRunId: runId, issueId: issue.id, provider: "daytona", providerLeaseId, status: "active", leasePolicy, metadata: { driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true, remoteCwd: "/work" } }); + await db.insert(heartbeatRunEvents).values({ companyId: company.id, runId, agentId: agent.id, seq: 1, + eventType: "native.process_start_requested", stream: "system", level: "info", message: "Runner launch requested" }); const deps = callbacks(db), control = createHeartbeatRunControl(db, deps); let ready!: () => void, release!: () => void; const acquired = new Promise(resolve => { ready = resolve; }); @@ -883,7 +887,8 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat status: "cancelled", providerResourceDisposition: "stop_and_retain" }); expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]) - .toMatchObject({ status: "active", leasePolicy }); + .toMatchObject({ status: "active", leasePolicy, metadata: expect.objectContaining({ releaseDeferral: expect.objectContaining({ + reason: "workspace_retention_busy", attempts: 1, providerResourceDisposition: "stop_and_retain" }) }) }); } finally { release(); await holding; } await preserveCancelledNativeWorkspaces(db); expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]) @@ -898,6 +903,109 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].status).toBe("in_progress"); }, 15_000); + async function deferralFixture(runtimeMode: "legacy" | "native") { + const { company, agent } = await fixture(); + const runId = randomUUID(), leaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: company.id, agentId: agent.id, runtimeMode, + status: "cancelled", finishedAt: new Date() }); + await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, heartbeatRunId: runId, + provider: "daytona", providerLeaseId: randomUUID(), status: "active", leasePolicy: "ephemeral", + metadata: { driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true } }); + if (runtimeMode === "native") await db.insert(heartbeatRunEvents).values({ companyId: company.id, runId, agentId: agent.id, + seq: 1, eventType: "native.process_start_requested", stream: "system", level: "info", message: "Runner launch requested" }); + const deferral = async () => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]! + .metadata?.releaseDeferral as { reason: string; attempts: number; retryAt: string; providerResourceDisposition: string | null }; + return { company, agent, runId, deferral }; + } + + it("records a failed release with its disposition and a growing retry time", async () => { + const { company, agent, runId, deferral } = await deferralFixture("legacy"); + const deps = callbacks(db); + vi.mocked(deps.envOrchestrator.releaseForRun).mockResolvedValue({ released: [], errors: [{ leaseId: "*", error: new Error("provider down") }] }); + const control = createHeartbeatRunControl(db, deps); + const before = Date.now(); + await control.releaseEnvironmentLeasesForRun({ runId, companyId: company.id, agentId: agent.id, status: "cancelled" }); + const first = await deferral(); + expect(first).toMatchObject({ reason: "release_failed", attempts: 1, providerResourceDisposition: null }); + expect(Date.parse(first.retryAt) - before).toBeGreaterThanOrEqual(30_000); + expect(Date.parse(first.retryAt) - before).toBeLessThan(40_000); + await control.releaseEnvironmentLeasesForRun({ runId, companyId: company.id, agentId: agent.id, status: "cancelled" }); + const second = await deferral(); + expect(second).toMatchObject({ reason: "release_failed", attempts: 2 }); + expect(Date.parse(second.retryAt) - before).toBeGreaterThanOrEqual(60_000); + }); + + it("never gives up on a failing release: hourly retries after the fast backoff, then release and wake the stopping agent", async () => { + const { company, agent, runId, deferral } = await deferralFixture("legacy"); + const leaseId = (await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, runId)))[0]!.id; + await db.update(agents).set({ lifecycleState: "terminating", lifecycleOperation: { id: randomUUID(), hostComplete: false, + completedPluginIds: [], attempts: 3, retryAt: new Date(Date.now() + 60_000).toISOString() } }).where(eq(agents.id, agent.id)); + const deps = callbacks(db); + let outageCalls = 0; + vi.mocked(deps.envOrchestrator.releaseForRun).mockImplementation(async () => { + outageCalls += 1; + return { released: [], errors: [{ leaseId: "*", error: new Error("provider down") }] }; + }); + const control = createHeartbeatRunControl(db, deps); + const fastSteps = LEASE_RELEASE_FAST_ATTEMPTS; + const dueRetriesInThreeHours = fastSteps + 3; + let firstFailure: string | undefined; + for (let attempt = 1; attempt <= dueRetriesInThreeHours; attempt++) { + const before = Date.now(); + await control.releaseEnvironmentLeasesForRun({ runId, companyId: company.id, agentId: agent.id, status: "cancelled" }); + const current = await deferral() as Record & { retryAt: string }; + firstFailure ??= current.failingSince as string; + expect(current).toMatchObject({ reason: "release_failed", attempts: attempt, failingSince: firstFailure }); + const delay = Date.parse(current.retryAt) - before; + if (attempt > fastSteps) expect(delay).toBeGreaterThanOrEqual(LEASE_RELEASE_SLOW_RETRY_MS - 1_000); + else expect(delay).toBeLessThanOrEqual(15 * 60_000 + 1_000); + } + expect(outageCalls).toBe(dueRetriesInThreeHours); + expect(await agentExecutionStopBlocker(db, [agent.id])).toBe("environment_lease_release_deferred:release_failed"); + vi.mocked(deps.envOrchestrator.releaseForRun).mockImplementation(async () => { + await db.update(environmentLeases).set({ status: "expired", releasedAt: new Date(), cleanupStatus: "success" }) + .where(eq(environmentLeases.id, leaseId)); + return { released: [{} as never], errors: [] }; + }); + await control.releaseEnvironmentLeasesForRun({ runId, companyId: company.id, agentId: agent.id, status: "cancelled" }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]!.status).toBe("expired"); + expect((await db.select().from(agents).where(eq(agents.id, agent.id)))[0]!.lifecycleOperation).not.toHaveProperty("retryAt"); + }); + + it("records a destroy held back by a busy warm session", async () => { + const { company, agent, runId, deferral } = await deferralFixture("native"); + const deps = callbacks(db); + deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 }); + await createHeartbeatRunControl(db, deps).releaseEnvironmentLeasesForRun({ runId, companyId: company.id, + agentId: agent.id, status: "cancelled", providerResourceDisposition: "destroy" }); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(await deferral()).toMatchObject({ reason: "warm_session_busy", attempts: 1, providerResourceDisposition: "destroy" }); + }); + + it("stops and retains a launched cancelled run's sandbox when no file retention is owed", async () => { + const { company, agent } = await fixture(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Stopped task", + status: "in_progress", assigneeAgentId: agent.id }).returning(); + const runId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: company.id, agentId: agent.id, + issueId: issue.id, nativeIssueId: issue.id, runtimeMode: "native", status: "cancelled", finishedAt: new Date(), + runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", + descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, + workspaceId: randomUUID(), leaseId, providerLeaseId, remoteCwd: "/work", resourceDisposition: "destroy" } } }); + await db.insert(nativeRunFinalizations).values({ runId, companyId: company.id, issueId: issue.id, + phase: "terminal_failure", failureCode: "native_retry_cancelled" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, heartbeatRunId: runId, + issueId: null, provider: "daytona", providerLeaseId, status: "active", leasePolicy: "ephemeral", + metadata: { driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true, remoteCwd: "/work" } }); + await db.insert(heartbeatRunEvents).values({ companyId: company.id, runId, agentId: agent.id, seq: 1, + eventType: "native.process_start_requested", stream: "system", level: "info", message: "Runner launch requested" }); + const deps = callbacks(db); + await createHeartbeatRunControl(db, deps).releaseEnvironmentLeasesForRun({ runId, companyId: company.id, + agentId: agent.id, status: "cancelled", providerResourceDisposition: "destroy" }); + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ + heartbeatRunId: runId, providerResourceDisposition: "stop_and_retain" })); + }); + it("reports a missing run without entering cancellation", async () => { const deps = callbacks(db); await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found"); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index ccc2a21b872..3b26f4f01cc 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -7,6 +7,10 @@ import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifi import { resumeDeferredChatWake } from "../durable-chat-wakeup.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { readNativeStopExecutionEvidence } from "./native-stop-evidence-read.js"; +import { LEASE_RELEASE_DEFERRAL_KEY, nativeRunNeverLaunchedRunner, readLeaseReleaseDeferral, + settleUnchangedNativeWorkspace, wakeStoppingAgentLifecycle, nextLeaseReleaseTriggers, nextLeaseReleaseBootIds, nextLeaseReleaseRetryAt, + type LeaseReleaseDeferral } from "../environment-lease-release-deferral.js"; +import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js"; import { readTaskRetryIntent, reconcileTaskChatRetryReceipts } from "../task-retry.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, readNativeLocalProcessStop, recordedLocalExecutionStopped } from "../native-local-process-stop.js"; import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; @@ -20,8 +24,9 @@ import { type WakeupOptions, } from "./run-preparation.js"; import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; -import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; +import { nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; import { preserveNativeFailedWorkspace } from "../native-runtime/native-failed-workspace-recovery.js"; +import { NativeWorkspaceFinalizationBusyError } from "../native-runtime/native-workspace-finalization-ownership.js"; import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js"; import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; import { @@ -182,7 +187,7 @@ async function withBudgetStopDelivery(db: Db, work: () => Promise): Promis /** `preservePauseDeferred`: the stop comes from a pause, which keeps the * messages it deferred for delivery once the pause lifts. */ -export type StopInvocationOptions = { preservePauseDeferred?: boolean }; +export type StopInvocationOptions = { preservePauseDeferred?: boolean; trigger?: string }; /** Cancellation shares executor ownership and delegates lifecycle and admission effects. */ @@ -382,6 +387,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont failureReason?: string | null; providerResourceDisposition?: ProviderResourceDisposition; respectCleanupDeadlines?: boolean; + trigger?: string; nativeLifecycleTelemetry?: { provider: string; harness: string; @@ -402,21 +408,34 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont input.providerResourceDisposition, status, ); + let neverLaunched: boolean | undefined; + const runnerNeverLaunched = async () => neverLaunched ??= leaseOwnerRun?.runtimeMode === "native" + && isHeartbeatRunTerminalStatus(status) && await nativeRunNeverLaunchedRunner(db, input.runId, input.companyId); + const defer = (reason: string) => deferEnvironmentLeaseRelease(input, reason, providerResourceDisposition); if (cancelledRun?.runtimeMode === "native" && status === "cancelled" && readNativeWorkspaceSyncReference(cancelledRun.runnerProfileJson?.nativeWorkspaceSync)?.state === "prepared" - && !nativeRunHasNoWorkspaceAccess(cancelledRun)) { - try { - const preserved = await preserveNativeFailedWorkspace(db, cancelledRun, { allowPreterminal: true }); - if (!preserved || !hasRequiredWorkspaceRecovery(preserved.resultJson)) return; - } catch { + && !nativeRunHasNoWorkspaceAccess(cancelledRun) && !await runnerNeverLaunched()) { + let deferral: string | null = null; + for (let attempt = 0; attempt < 2; attempt++) { + try { + await preserveNativeFailedWorkspace(db, cancelledRun, { allowPreterminal: true }); + deferral = null; + break; + } catch (error) { + const busy = error instanceof NativeWorkspaceFinalizationBusyError; + deferral = busy ? "workspace_owner_active" : "workspace_retention_busy"; + if (!busy || attempt > 0 || !await clearExpiredNativeFinalizationOwner(input.runId, input.companyId)) break; + } + } + if (deferral) { // Keep the active source eligible for the bounded retention sweep. // Teardown must not release it after retention lost a lock or DB write. logger.warn({ runId: input.runId }, "Deferred cancelled workspace teardown until file retention is recorded"); - return; + return await defer(deferral); } providerResourceDisposition = "stop_and_retain"; } - if (providerResourceDisposition === "destroy") { + if (providerResourceDisposition === "destroy" && !await runnerNeverLaunched()) { const closeResult = await ( options.closeWarmNativeSessionsForRun ?? (async ({ runId, reason }) => { @@ -476,7 +495,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont { runId: input.runId, warmNativeSessions: closeResult }, "deferred environment lease destruction until warm native sessions close", ); - return; + return await defer(closeResult.busy > 0 ? "warm_session_busy" : "warm_session_close_failed"); } } const releaseResult = await envOrchestrator @@ -507,9 +526,71 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont "failed to release environment lease for heartbeat run", ); } + if (isHeartbeatRunTerminalStatus(status) && (!releaseResult || releaseResult.errors.length > 0)) await defer("release_failed"); + else if (isHeartbeatRunTerminalStatus(status) && leaseOwnerRun?.runtimeMode === "native") await defer("release_skipped"); + if (cancelledRun && neverLaunched + && readNativeWorkspaceSyncReference(cancelledRun.runnerProfileJson?.nativeWorkspaceSync)?.state === "prepared") { + await settleUnchangedNativeWorkspace(db, input.runId, input.companyId).catch(() => { + logger.warn({ runId: input.runId }, "could not record that a stopped run left no files to recover"); + }); + } + if (isHeartbeatRunTerminalStatus(status) && releaseResult?.released.length) await wakeAgentAfterRunLeasesSettled(input).catch(() => { + logger.warn({ runId: input.runId }, "could not wake the agent's stop after its run leases settled"); + }); await acknowledgeRemoteStop(input.runId, input.companyId); } + async function wakeAgentAfterRunLeasesSettled(input: { runId: string; companyId: string; agentId: string }) { + const [unsettled] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.heartbeatRunId, input.runId), + inArray(environmentLeases.status, ["active", "pending_cleanup"]))).limit(1); + if (!unsettled) await db.transaction(tx => wakeStoppingAgentLifecycle(tx as unknown as Db, input.agentId)); + } + + async function clearExpiredNativeFinalizationOwner(runId: string, companyId: string) { + const cleared = await db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: new Date() }) + .where(and(eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.companyId, companyId), + isNotNull(nativeRunFinalizations.leaseOwner), lte(nativeRunFinalizations.leaseExpiresAt, new Date()), + isNull(nativeRunFinalizations.resultId), + sql`exists (select 1 from ${heartbeatRuns} where ${heartbeatRuns.id} = ${runId} + and ${heartbeatRuns.companyId} = ${companyId} and ${heartbeatRuns.status} = 'cancelled')`)) + .returning({ runId: nativeRunFinalizations.runId }); + return cleared.length > 0; + } + + async function deferEnvironmentLeaseRelease(input: { runId: string; companyId: string; trigger?: string }, reason: string, + providerResourceDisposition: ProviderResourceDisposition | undefined) { + const leases = await Promise.resolve().then(() => db.select({ id: environmentLeases.id, metadata: environmentLeases.metadata }) + .from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.status, "active")))) + .catch(() => { + logger.warn({ runId: input.runId, reason }, "could not record a deferred environment lease release"); + return []; + }); + for (const lease of leases) { + const previous = readLeaseReleaseDeferral(lease.metadata); + const attempts = (previous?.attempts ?? 0) + 1; + const deferral: LeaseReleaseDeferral = { reason, attempts, + retryAt: nextLeaseReleaseRetryAt(previous, attempts), + failingSince: previous?.failingSince ?? new Date().toISOString(), + triggers: nextLeaseReleaseTriggers(previous, input.trigger), attemptedByBootIds: nextLeaseReleaseBootIds(previous), + providerResourceDisposition: providerResourceDisposition ?? null }; + const recorded = await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.orphanActiveLeases, + JSON.stringify({ companyId: input.companyId, runId: input.runId, leaseId: lease.id })); + await tx.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) + || ${JSON.stringify({ [LEASE_RELEASE_DEFERRAL_KEY]: deferral })}::jsonb` }) + .where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.status, "active"))); + }).then(() => true, () => false); + if (!recorded) { + logger.warn({ runId: input.runId, leaseId: lease.id, reason }, "could not record a deferred environment lease release"); + continue; + } + logger.warn({ runId: input.runId, leaseId: lease.id, reason, attempts, retryAt: deferral.retryAt, + failingSince: deferral.failingSince }, "deferred environment lease release; a background pass retries it"); + } + } + async function acknowledgeRemoteStop(runId: string, companyId: string) { // The provider receipt arrives after adapter settlement. A remote ACP child // has no host PID, so only this target-aware boundary can acknowledge Stop. @@ -1728,12 +1809,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont errorCode = "cancelled", ) { const runs = await db - .select() + .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + or(inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + pendingFailedNativeRetryCancellationCondition()), ), ); diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation-idle.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation-idle.test.ts new file mode 100644 index 00000000000..34ebec77d61 --- /dev/null +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation-idle.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from "node:crypto"; +import { eq, inArray } from "drizzle-orm"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, + nativeRunFinalizations } from "@paperclipai/db"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { registerCancelledWorkspacePreservationWork } from "./native-failed-workspace-recovery.js"; + +(recoveryPostgresSupport.supported ? describe : describe.skip)("cancelled workspace preservation idle cost", () => { + let database: Awaited>; + let db: ReturnType; + const fixtures = new Set(); + + async function seed(input: { launched: boolean; lease: "destroyed" | "active" }) { + if (!database) { + database = await startRecoveryPostgres("cancelled-workspace-idle-"); + db = createDb(database.connectionString); + } + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const leaseId = randomUUID(), environmentId = randomUUID(), providerLeaseId = `allocation-${randomUUID()}`; + fixtures.add(runId); + await db.insert(companies).values({ id: companyId, name: "Stopped native files", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Stopped worker", adapterType: "paperclip_runner", + status: "terminated", lifecycleState: "terminating" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Stopped task", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(environments).values({ id: environmentId, name: `Idle ${runId}`, driver: "sandbox" }); + const nativeCancellation = { schema: "paperclip.native-cancellation.v1", companyId, issueId, runId, + scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true, + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID(), acknowledgedAt: new Date().toISOString(), + effects: ["release_run_resources"] }; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, nativeIssueId: issueId, + runtimeMode: "native", runtimeModeResolvedAt: new Date(), status: "cancelled", errorCode: "cancelled", + error: "Agent lifecycle stop requested", finishedAt: new Date(), contextSnapshot: { issueId }, + nextEventSeq: input.launched ? 2 : 1, + runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", + descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, + workspaceId: randomUUID(), leaseId, providerLeaseId, remoteCwd: "/work", resourceDisposition: "destroy" } }, + resultJson: { nativeCancellation, startupCancellation: { beforeNativeSelection: false } } }); + await db.insert(nativeRunFinalizations).values(input.launched + ? { runId, companyId, issueId, phase: "terminal_failure", failureCode: "native_retry_cancelled" } + : { runId, companyId, issueId, phase: "observed", attempt: 1, controllerGeneration: 1, + leaseOwner: `executor-${runId}`, leaseExpiresAt: new Date(Date.now() - 60_000) }); + if (input.launched) await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1, + eventType: "native.process_start_requested", stream: "system", level: "info", message: "Runner launch requested" }); + const destroyed = input.lease === "destroyed"; + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId, issueId, + provider: "daytona", providerLeaseId, leasePolicy: "ephemeral", status: destroyed ? "expired" : "active", + releasedAt: destroyed ? new Date() : null, cleanupStatus: destroyed ? "success" : null, + metadata: { driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true, remoteCwd: "/work", + ...(destroyed ? { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", state: "destroyed", + runId, leaseId, companyId, provider: "daytona", providerLeaseId, confirmedAt: new Date().toISOString() } } : {}) } }); + return { companyId, runId, leaseId }; + } + + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + afterEach(async () => { + vi.useRealTimers(); vi.restoreAllMocks(); + if (fixtures.size) await db.update(heartbeatRuns).set({ status: "failed" }).where(inArray(heartbeatRuns.id, [...fixtures])); + fixtures.clear(); + }); + + async function idleStatements(minutes: number) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), writes = vi.spyOn(db, "update"); + await vi.advanceTimersByTimeAsync(minutes * 60_000); + const count = reads.mock.calls.length + transactions.mock.calls.length + writes.mock.calls.length; + reads.mockRestore(); transactions.mockRestore(); writes.mockRestore(); vi.useRealTimers(); + return count; + } + + it("leaves a stopped run that never started a runner alone, before and after a restart", async () => { + const f = await seed({ launched: false, lease: "destroyed" }); + for (let boot = 0; boot < 2; boot++) { + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const checks = vi.spyOn(db, "update"); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + expect(checks.mock.calls.length).toBe(boot === 0 ? 1 : 0); + checks.mockRestore(); + expect(worker.nextWakeAt()).toBeNull(); + expect(await idleStatements(30)).toBe(0); + } finally { checks.mockRestore(); await worker.stop(); } + } + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.runnerProfileJson).not.toHaveProperty("nativeWorkspaceSync"); + expect(run.runnerProfileJson).toHaveProperty("nativeWorkspaceSyncReleasedUnchanged.state", "prepared"); + expect(run.resultJson).not.toHaveProperty("workspaceRestoreRecovery"); + }, 120_000); + + it("still preserves the files of a stopped run that started a runner, then goes idle", async () => { + const f = await seed({ launched: true, lease: "active" }); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.resultJson).toMatchObject({ workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", + leaseIds: [f.leaseId] } }); + expect(worker.nextWakeAt()).toBeNull(); + expect(await idleStatements(30)).toBe(0); + } finally { await worker.stop(); } + }, 120_000); +}); diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index f693698265d..54de27f1887 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -9,6 +9,7 @@ import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recov import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; import { logActivity } from "../activity-log.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; +import { nativeRunNeverLaunchedRunner, settleUnchangedNativeWorkspace } from "../environment-lease-release-deferral.js"; import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finalization-ownership.js"; import { hasAcknowledgedNativeRunStopRequest } from "../acknowledged-native-stop.js"; @@ -329,9 +330,16 @@ export function registerCancelledWorkspacePreservationWork(db: Db, eq(nativeRunFinalizations.issueId, run.nativeIssueId!), )); if (finalization?.resultId && !stoppedAcceptedCopyback(finalization)) return null; - if (finalization?.phase !== "terminal_failure" || finalization.leaseOwner) + if (finalization?.phase !== "terminal_failure" || finalization.leaseOwner) { + const ownerDeadline = finalization?.leaseOwner ? finalization.leaseExpiresAt?.getTime() ?? null : null; + if (ownerDeadline && ownerDeadline > Date.now()) return { ...entry, dueAt: ownerDeadline }; + if (await nativeRunNeverLaunchedRunner(db, run.id, run.companyId) + && await settleUnchangedNativeWorkspace(db, run.id, run.companyId)) return null; return { ...entry, dueAt: Date.now() + 30_000 }; + } if (await cancelledWorkspaceSource(db, run)) await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true }); + else if (await nativeRunNeverLaunchedRunner(db, run.id, run.companyId) + && await settleUnchangedNativeWorkspace(db, run.id, run.companyId)) return null; const [current] = await db.select().from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId))); const currentControllerDeadline = current?.controllerLeaseExpiresAt?.getTime(); diff --git a/server/src/services/native-runtime/native-workspace-finalization-state.ts b/server/src/services/native-runtime/native-workspace-finalization-state.ts index 1921029dafb..430096da99d 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-state.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-state.ts @@ -1,4 +1,5 @@ -import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { environmentLeases, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { executionStartedEventPredicate } from "../pre-dispatch-execution-evidence.js"; import { and, eq, exists, sql } from "drizzle-orm"; import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recovery.js"; @@ -63,6 +64,10 @@ function nativeWorkspaceRunProjection() { 'schema', ${input}->'schema', 'binding', ${input}->'binding', 'workspace', ${input}->'workspace', 'provider', jsonb_build_object('kind', ${input}#>'{provider,kind}', 'bridgeRevision', ${input}#>'{provider,bridgeRevision}'))`, controllerBootId: heartbeatRuns.controllerBootId, + cancelledBeforeRunnerLaunch: sql`${heartbeatRuns.status} = 'cancelled' + and ${heartbeatRuns.processPid} is null and ${heartbeatRuns.processGroupId} is null + and not exists (select 1 from ${heartbeatRunEvents} where ${and(eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), + eq(heartbeatRunEvents.runId, heartbeatRuns.id), executionStartedEventPredicate())})`, contextIssueId: sql`${heartbeatRuns.contextSnapshot}->>'issueId'`, remoteReference: sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'`, }; @@ -71,9 +76,11 @@ function nativeWorkspaceRunProjection() { function nativeWorkspaceSourcePending(run: Pick & { reference: unknown; executionScope: Record; observedRemoteWorkspace: boolean; + cancelledBeforeRunnerLaunch: boolean; }, lease: WorkspaceSourceLease) { if (run.runtimeMode !== "native") return false; if (run.observedRemoteWorkspace) return true; + if (run.cancelledBeforeRunnerLaunch) return false; if (nativeRunHasNoWorkspaceAccess({ ...run, runnerProfileJson: { nativeExecutionInput: run.executionScope } })) return false; const reference = readNativeWorkspaceSyncReference(run.reference); return reference?.state === "prepared" && reference.leaseId === lease.id