diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index f3103ca9e77..dbfef487ba4 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -325,6 +325,7 @@ const nativeRunnerSuitePath = const ingressRetrySuitePath = "server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts"; const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts"; const warmCheckpointOwnershipSuitePath = "server/src/services/native-runtime/native-warm-checkpoint-ownership.integration.test.ts"; +const warmSuccessorRunnerReleaseSuitePath = "server/src/services/native-runtime/native-warm-successor-runner-release.integration.test.ts"; const pluginWarmOwnerSuitePath = "server/src/services/native-runtime/plugin-runner-task-warm.integration.test.ts"; const pluginReleaseResumeSuitePath = "server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts"; const pluginEvictionSuitePath = "server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts"; @@ -378,10 +379,11 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes assert.ok(!files.includes(cutTurnStartRetrySuitePath)); assert.ok(!files.includes(usageDeliverySuitePath)); assert.ok(!files.includes(warmCheckpointOwnershipSuitePath)); + assert.ok(!files.includes(warmSuccessorRunnerReleaseSuitePath)); assert.ok(!files.includes(pluginWarmOwnerSuitePath)); assert.ok(!files.includes(pluginReleaseResumeSuitePath)); assert.ok(!files.includes(pluginEvictionSuitePath)); - assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, ingressRetrySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, ...restartWaitSuites, cutTurnStartRetrySuitePath, usageDeliverySuitePath, warmCheckpointOwnershipSuitePath, pluginWarmOwnerSuitePath, pluginReleaseResumeSuitePath, pluginEvictionSuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); + assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, ingressRetrySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, ...restartWaitSuites, cutTurnStartRetrySuitePath, usageDeliverySuitePath, warmCheckpointOwnershipSuitePath, warmSuccessorRunnerReleaseSuitePath, pluginWarmOwnerSuitePath, pluginReleaseResumeSuitePath, pluginEvictionSuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); assert.equal(new Set(files).size, files.length); const defaultRun = dryRunJson([], prEnv); assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); @@ -402,6 +404,7 @@ for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" } assert.ok(files.includes(dotRunnerSuitePath)); assert.ok(files.includes(restartRecoverySuitePath)); assert.ok(files.includes(warmCheckpointOwnershipSuitePath)); + assert.ok(files.includes(warmSuccessorRunnerReleaseSuitePath)); assert.ok(files.includes(pluginWarmOwnerSuitePath)); assert.ok(files.includes(pluginReleaseResumeSuitePath)); assert.ok(files.includes(pluginEvictionSuitePath)); @@ -439,6 +442,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => { cutTurnStartRetrySuitePath, usageDeliverySuitePath, warmCheckpointOwnershipSuitePath, + warmSuccessorRunnerReleaseSuitePath, pluginWarmOwnerSuitePath, pluginReleaseResumeSuitePath, pluginEvictionSuitePath, diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 42be01a33d0..473403a99cc 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -107,6 +107,7 @@ const nativeRunnerSuites = [ "server/src/services/native-runtime/native-cut-turn-start-retry.integration.test.ts", "server/src/services/native-runtime/native-usage-delivery.integration.test.ts", "server/src/services/native-runtime/native-warm-checkpoint-ownership.integration.test.ts", + "server/src/services/native-runtime/native-warm-successor-runner-release.integration.test.ts", "server/src/services/native-runtime/plugin-runner-task-warm.integration.test.ts", "server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts", "server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts", diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c0011b7afc0..af41f34a222 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -13313,7 +13313,6 @@ describe("native warm session supervision", () => { runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId, remoteCwd: `/tmp/${environmentId}`, sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "gone-sandbox" } } }); const deleted = await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true, db }); - expect(where).toHaveBeenCalledOnce(); if (receipt === "destroyed") { expect(deleted).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true, db })) diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c16a79bc208..cb068a80ddf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1127,12 +1127,40 @@ async function warmSandboxTerminated(db: Db, entry: WarmNativeSession): Promise< }); } -/** A failed close is complete when its Runner is proven gone, its sandbox is destroyed, or the run's own sandbox lease has a stop receipt. */ +async function warmRunnerPortTakenBySuccessor(db: Db, entry: WarmNativeSession): Promise { + if (!entry.providerLeaseId || !entry.environmentId) return false; + const [owner] = await db.select({ finishedAt: heartbeatRuns.finishedAt, + lastRunnerEventAt: sql`(select max(owner_event.created_at)::text from ${heartbeatRunEvents} owner_event + where owner_event.company_id = ${entry.companyId} and owner_event.run_id = ${entry.runId} + and owner_event.source_event_id is not null)` }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, entry.runId), eq(heartbeatRuns.companyId, entry.companyId))); + if (!owner?.finishedAt || !owner.lastRunnerEventAt) return false; + const ownerLastRunnerEventAt = owner.lastRunnerEventAt; + const [successor] = await db.select({ runId: heartbeatRuns.id }).from(environmentLeases) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, entry.companyId))) + .where(and( + eq(environmentLeases.providerLeaseId, entry.providerLeaseId), + eq(environmentLeases.companyId, entry.companyId), + eq(environmentLeases.environmentId, entry.environmentId), + sql`${heartbeatRuns.id} <> ${entry.runId}`, + sql`exists (select 1 from ${heartbeatRunEvents} launch + where launch.company_id = ${entry.companyId} and launch.run_id = ${heartbeatRuns.id} + and launch.event_type = ${PROCESS_IDENTITY_RECORDED} and launch.source_event_id is null + and launch.created_at > ${ownerLastRunnerEventAt}::timestamptz)`, + sql`exists (select 1 from ${heartbeatRunEvents} started + where started.company_id = ${entry.companyId} and started.run_id = ${heartbeatRuns.id} + and started.event_type in ('session.started', 'session.resumed') and started.source_instance_id = ${heartbeatRuns.runnerInstanceId}::text + and started.created_at > ${ownerLastRunnerEventAt}::timestamptz)`, + )).limit(1); + return Boolean(successor); +} + +/** A failed close is complete when its Runner is proven gone, its sandbox is destroyed, the run's own sandbox lease has a stop receipt, or a later run launched its own Runner on the same sandbox. */ async function warmCloseFailureProvesOwnerGone(entry: WarmNativeSession, error: unknown, db = entry.sandboxLeaseDb): Promise { const diagnostic = readRemoteRunnerRecoveryError(error); if (diagnostic?.stage === "process_verification" && diagnostic.reason === "process_not_running") return true; try { - return db ? await warmSandboxTerminated(db, entry) : false; + return db ? await warmSandboxTerminated(db, entry) || await warmRunnerPortTakenBySuccessor(db, entry) : false; } catch (proofError) { logger.warn({ err: proofError, runId: entry.runId }, "could not read the warm session's sandbox lease"); return false; diff --git a/server/src/services/native-runtime/native-warm-successor-runner-release.integration.test.ts b/server/src/services/native-runtime/native-warm-successor-runner-release.integration.test.ts new file mode 100644 index 00000000000..6a13bf4001a --- /dev/null +++ b/server/src/services/native-runtime/native-warm-successor-runner-release.integration.test.ts @@ -0,0 +1,197 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { completionContracts, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, + nativeRunnerCheckpoints } from "@paperclipai/db"; +import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { createStorageService } from "../../storage/service.js"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../../vendor/paperclip-runner/testing.js"; +import { defaultCapabilityRunnerdBinary } from "../../vendor/paperclip-runner/index.js"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "../native-process-start-events.js"; + +type ProviderTransport = ReturnType[1]>["codexTransportFactory"]>>; +const { FakeCodexTransport } = await import(new URL("../../../../packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts", import.meta.url).href) as { FakeCodexTransport: new () => ProviderTransport }; +const { runnerdRecoveryInternals } = await import(new URL("../../../../packages/paperclip-runner/src/live/runnerd-codex-transport.ts", import.meta.url).href) as { + runnerdRecoveryInternals: { releaseRunnerProcessOwnership(input: { runnerSettled: boolean; checkpoint: typeof state.checkpoint; + release: () => Promise; forceKill: () => Promise }): Promise }; +}; + +const state = vi.hoisted(() => ({ execute: vi.fn(), checkpoint: null as null | ((settlement: "settled" | "unsettled") => Promise), storage: null as unknown, transport: null as unknown })); +vi.mock("./native-harness-backup-storage.js", async original => ({ + ...await original(), + durableNativeHarnessStorage: () => state.storage, +})); +vi.mock("./native-runner-execution-driver.js", async original => { + const actual = await original(); + return { ...actual, createNativeRunnerExecutionDriver: (input: Parameters[0]) => { + if (input.checkpoint) state.checkpoint = input.checkpoint as typeof state.checkpoint; + return actual.createNativeRunnerExecutionDriver(input); + } }; +}); +vi.mock("../../vendor/paperclip-runner/index.js", async original => { + const actual = await original(); + return { ...actual, executeNativeSession: state.execute, + createNativeSessionBackend: (input: Parameters[0], options: Parameters[1]) => { + options?.codexTransportFactory?.(); + return actual.createNativeSessionBackend(input, { ...options, codexTransportFactory: () => state.transport as ProviderTransport }); + } }; +}); +import { closeIdleWarmNativeSessionsForRestart, closeWarmNativeSessionsForEnvironment, executePaperclipNativeSession, + readNativeSessionIdleWork } from "./native-session-executor.js"; +import { assertNativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; + +type Successor = "own_runner" | "owner_clock_ahead" | "own_runner_resumed" | "own_runner_reconciled" | "other_sandbox" | "start_requested_only" | "inherited_events" | "events_before_owner_finished"; + +describeEmbeddedPostgres("idle warm owner superseded by a successor's Runner", () => { + const environment = nativeRestartFixture(); + let root: string; + beforeAll(async () => { root = await mkdtemp(join(tmpdir(), "warm-successor-runner-")); }); + afterAll(async () => { state.storage = null; await rm(root, { recursive: true, force: true }); }); + + async function idleOwnerSupersededBySuccessor(label: string, successorKind: Successor, idleTimeoutMs: number) { + const fixture = await environment.seedRun(label); + const { companyId, agentId } = environment.current; + const [contract] = await fixture.db.select().from(completionContracts).where(eq(completionContracts.issueId, fixture.issueId)); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, executionWorkspaceId: randomUUID() }, + provider: { kind: "codex", model: "test-model" }, + task: { identifier: "WARM-2", title: "Idle warm owner", description: null, prompt: "Finish and stay warm.", workMode: "standard" }, + workspace: { cwd: root, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: fixture.native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "warm", idleTimeoutMs } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: { ...contract.contractJson, revision: String(contract.revision) } } as NativeExecutionInputV1["completionContract"], + interactionResponses: [], credentialBindings: [], + }; + state.storage = createStorageService(createLocalDiskStorageProvider(join(root, "objects"))); + const transport = new FakeCodexTransport(); + const request = transport.request.bind(transport); + transport.request = async (method, params) => { + const response = await request(method, params); + return method === "thread/start" || method === "thread/resume" + ? { ...response, cwd: root, thread: { ...response.thread as Record, cwd: root } } + : response; + }; + state.transport = transport; + const closeFailures: string[] = []; + transport.close = async () => { + try { + await runnerdRecoveryInternals.releaseRunnerProcessOwnership({ runnerSettled: false, + checkpoint: state.checkpoint, release: async () => {}, forceKill: async () => {} }); + } catch (error) { closeFailures.push(String(error)); throw error; } + }; + state.execute.mockReset().mockImplementation(async options => { + const session = await options.backend.openSession({ identity: { companyId, agentId, issueId: fixture.issueId, + runId: fixture.runId, sessionId: execution.session.normalizedSessionId }, workingDirectory: root }); + await options.onSession(session); + await options.controlPlane.openRun({ identity: { companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, + sessionId: execution.session.normalizedSessionId }, backendKind: "mock", sourceInstanceId: fixture.native.runnerInstanceId }); + const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, + completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: execution.completionContract.contract.revision } }; + await options.controlPlane.completeRun({ result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: fixture.native.turnId }); + return { result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: fixture.native.turnId, normalizedSessionId: execution.session.normalizedSessionId, + providerSessionId: "owner-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 0, highestContiguousSourceSeq: 0, usage: null }; + }); + const environmentId = randomUUID(); + const providerLeaseId = randomUUID(); + await fixture.db.insert(environments).values({ id: environmentId, name: `Sandbox ${label}`, driver: "sandbox" }); + await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.preparedProfile, nativeExecutionInput: execution } }) + .where(eq(heartbeatRuns.id, fixture.runId)); + const owner = await executePaperclipNativeSession({ db: fixture.db, execution, runnerInstanceId: fixture.native.runnerInstanceId, + useRunnerd: true, runnerIngressAuthorized: true, + runnerRemoteBinaryPath: defaultCapabilityRunnerdBinary(), runnerRemoteCodexPath: defaultCapabilityRunnerdBinary(), + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: root, environmentId, + leaseId: randomUUID(), providerKey: "fixture", effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId }, + runner: { execute: async (command: { args?: string[] }) => command.args?.[1] === "uname -s; uname -m" + ? { exitCode: 0, timedOut: false, stdout: `${process.platform === "darwin" ? "Darwin" : "Linux"}\n${process.arch === "arm64" ? "arm64" : "x86_64"}\n`, stderr: "" } + : command.args?.[2] === "paperclip-runner-port-holders" + ? { exitCode: 0, timedOut: false, stdout: "", stderr: "" } + : { exitCode: 1, timedOut: false, stdout: "", stderr: "" }, syncIn: async () => {} } } as never, + instructionWorkingCopy: { runId: fixture.runId, root, checkpointWarm: async () => true, collectStopped: async () => {}, hasChanges: async () => false }, + }); + expect(owner.exitCode).toBe(0); + expect(readNativeSessionIdleWork()).toBe("present"); + await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 1_000_000, eventType: "turn.completed", + sourceInstanceId: fixture.native.runnerInstanceId, sourceEventId: `owner-terminal-${fixture.runId}`, sourceSeq: 1_000_000 }); + const ownerFinishedAt = new Date(Date.now() + (successorKind === "owner_clock_ahead" ? 10 * 60_000 : -1_000)); + await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: ownerFinishedAt }).where(eq(heartbeatRuns.id, fixture.runId)); + const [finalized] = await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null, leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)).returning(); + expect(finalized.resultId).not.toBeNull(); + const successorRunId = randomUUID(), successorRunner = randomUUID(); + await fixture.db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: fixture.issueId, invocationSource: "automation", triggerDetail: "system", contextSnapshot: { issueId: fixture.issueId }, + runnerInstanceId: successorRunner, startedAt: new Date() }); + await fixture.db.update(issues).set({ executionRunId: successorRunId }).where(eq(issues.id, fixture.issueId)); + await fixture.db.insert(nativeRunFinalizations).values({ runId: successorRunId, companyId, issueId: fixture.issueId, phase: "observed", + controllerBootId: randomUUID() }); + await fixture.db.insert(environmentLeases).values({ companyId, environmentId, issueId: fixture.issueId, heartbeatRunId: successorRunId, + status: "active", leasePolicy: "reuse_by_environment", provider: "fixture", + providerLeaseId: successorKind === "other_sandbox" ? randomUUID() : providerLeaseId }); + const eventAt = successorKind === "events_before_owner_finished" ? new Date(Date.now() - 60_000) : undefined; + await fixture.db.insert(heartbeatRunEvents).values([ + { companyId, runId: successorRunId, agentId, seq: 1, createdAt: eventAt, + eventType: successorKind === "start_requested_only" ? PROCESS_START_REQUESTED : PROCESS_IDENTITY_RECORDED, + sourceEventId: successorKind === "inherited_events" ? `inherited-${randomUUID()}` : null }, + { companyId, runId: successorRunId, agentId, seq: 2, createdAt: eventAt, + eventType: successorKind === "own_runner_resumed" ? "session.resumed" : successorKind === "own_runner_reconciled" ? "session.reconciled" : "session.started", + sourceInstanceId: successorRunner, sourceEventId: `session-${successorRunId}`, sourceSeq: 1 }, + ]); + return { + closeFailures, + ownerCheckpoints: () => fixture.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, fixture.runId)), + assertOwner: () => fixture.db.transaction(tx => assertNativeRunnerCheckpointOwner(tx, { companyId, issueId: fixture.issueId, + runId: fixture.runId, authorityRunId: fixture.runId, bootId: finalized.controllerBootId!, generation: finalized.controllerGeneration, + leaseOwner: "" }, false)), + release: async () => { + await fixture.db.update(issues).set({ executionRunId: null, status: "done" }).where(eq(issues.id, fixture.issueId)); + await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture cleanup" }); + }, + }; + } + + it.each<{ successor: Successor; released: boolean }>([ + { successor: "own_runner", released: true }, + { successor: "own_runner_resumed", released: true }, + { successor: "owner_clock_ahead", released: true }, + { successor: "own_runner_reconciled", released: false }, + { successor: "other_sandbox", released: false }, + { successor: "start_requested_only", released: false }, + { successor: "inherited_events", released: false }, + { successor: "events_before_owner_finished", released: false }, + ])("releases an idle warm owner whose checkpoint is refused because a successor on another server launched its own Runner on the same sandbox: $successor", async ({ successor, released }) => { + const owner = await idleOwnerSupersededBySuccessor(`WARM-PORT-SUCCESSOR-${successor}`, successor, 60_000); + try { + const closed = await closeIdleWarmNativeSessionsForRestart(); + expect(owner.closeFailures).toEqual(["Error: native_runner_checkpoint_owner_lost"]); + if (released) expect(closed).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + else expect(closed).toMatchObject({ closed: 0, busy: 0, failed: 1 }); + expect(readNativeSessionIdleWork()).toBe(released ? "none" : "present"); + expect(await owner.ownerCheckpoints()).toEqual([]); + await expect(owner.assertOwner()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + } finally { + await owner.release(); + } + expect(readNativeSessionIdleWork()).toBe("none"); + }, 45_000); + + it("releases a superseded idle warm owner from its idle timer without writing a checkpoint", async () => { + const owner = await idleOwnerSupersededBySuccessor("WARM-PORT-SUCCESSOR-IDLE", "own_runner", 2_000); + try { + await vi.waitFor(() => expect(readNativeSessionIdleWork()).toBe("none"), { timeout: 10_000, interval: 100 }); + expect(owner.closeFailures).toEqual(["Error: native_runner_checkpoint_owner_lost"]); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); + expect(await owner.ownerCheckpoints()).toEqual([]); + await expect(owner.assertOwner()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + } finally { + await owner.release(); + } + }, 45_000); +});