diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index e03c1f8333..4bd3472ee3 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -28,6 +28,7 @@ import { inspect } from "node:util"; import { join } from "node:path"; import { heartbeatRuns, + environmentLeases, heartbeatRunEvents, issues, nativeRunFinalizations, @@ -82,6 +83,9 @@ vi.mock("node:fs", async (importOriginal) => { return { ...actual, readFileSync: vi.fn(actual.readFileSync) }; }); +const computerRetirement = vi.hoisted(() => ({ isRetired: vi.fn() })); +vi.mock("../../modules/computers/index.js", () => ({ computerService: () => computerRetirement })); + const githubAccess = vi.hoisted(() => ({ activate: vi.fn((_binding: { runId: string }) => vi.fn()), stop: vi.fn(async () => undefined), @@ -13438,6 +13442,65 @@ describe("runnerd provider runtime wiring", () => { } finally { await rm(root, { recursive: true, force: true }); } }); + it.each([ + { condition: "retired", expected: "verified" }, + { condition: "uncertain", expected: "terminal_state_indeterminate" }, + { condition: "stale-generation", expected: "terminal_state_indeterminate" }, + { condition: "wrong-placement", expected: "scope_mismatch" }, + { condition: "wrong-lease", expected: "scope_mismatch" }, + { condition: "active-run", expected: "active" }, + { condition: "unavailable", expected: "unavailable" }, + { condition: "corrupt-local", expected: "terminal_state_indeterminate" }, + ])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-")); + const prior = execution; + const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } }; + const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 }; + const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId, + remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation, + listenerPort: owner.listenerPort, placementId: "placement" }; + const db = { select: () => ({ from: (table: unknown) => ({ where: () => ({ limit: async () => + table === environmentLeases ? condition === "wrong-lease" ? [] : [{ + providerLeaseId: owner.ownerId, + metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } }, + }] : [{ status: condition === "active-run" ? "running" : "succeeded", + runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }], + }) }) }) } as unknown as Db; + computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => { + if (condition === "unavailable") throw new Error("ledger unavailable"); + return condition !== "uncertain" && requested.generation === (condition === "stale-generation" ? 3 : 2); + }); + const input = { db, root, execution: current, identity: { runId: prior.binding.runId, + normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner", + environmentLeaseId: condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" }, + allowVerifiedBackup: false, remoteRunnerState: true, + runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment", + remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 }, + fileAuthority: { placementId: condition === "wrong-placement" ? "other-placement" : "placement" } } as never }; + try { + if (condition === "corrupt-local") { + await mkdir(join(root, "runner")); + await writeFile(join(root, "runner", "runner-state.json"), "invalid state"); + } + if (condition === "retired") { + // Both live warm handoffs use their retained owner. After idle expiry, + // the next turn must instead prove the exact generation was retired. + for (let turn = 0; turn < 2; turn += 1) { + expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: true })) + .toBe("retained_warm_runner"); + } + expect(computerRetirement.isRetired).not.toHaveBeenCalled(); + } + expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected); + if (["retired", "uncertain", "stale-generation", "unavailable"].includes(condition)) { + expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({ + companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId, + runId: prior.binding.runId, owner, + }); + } else expect(computerRetirement.isRetired).not.toHaveBeenCalled(); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it.each([ { override: undefined, expected: "@openai/codex@0.160.0" }, { override: "@openai/codex@0.159.0", expected: "@openai/codex@0.159.0" }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c5bf41a377..d411b71e70 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,5 @@ +import { computerService } from "../../modules/computers/index.js"; +import { readNativeComputerWorkspaceReference } from "./native-workspace-sync.js"; import { adapterExecutionTargetIsCommandBacked, type AdapterComputerExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { configuredEnvironment } from "../../vendor/paperclip-runner/index.js"; import { agents } from "@paperclipai/db"; @@ -4295,6 +4297,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { allowVerifiedBackup: boolean; allowRetainedWarmRunner: boolean; remoteRunnerState?: boolean; + runnerExecutionTarget?: AdapterExecutionTarget | null; }): Promise { let priorRun: { status: string; @@ -4352,6 +4355,49 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { ) { return "retained_warm_runner"; } + const target = input.runnerExecutionTarget; + if (input.remoteRunnerState && directLifecycle === "absent" && + target?.kind === "remote" && target.transport === "computer" && target.environmentId) { + const descriptor = record(priorRun.runnerProfileJson).nativeComputerWorkspace; + const priorWorkspace = readNativeComputerWorkspaceReference(descriptor); + if (!priorWorkspace || priorWorkspace.computerOwner.computerId !== target.resourceAuthority.computerId || + priorWorkspace.remoteCwd !== target.remoteCwd || + record(descriptor).placementId !== target.fileAuthority.placementId) return "scope_mismatch"; + // Persistent runners keep their state remotely. Once the exact prior + // owner has completed retirement, its terminal run and fenced process + // are authoritative even though no local runner-state.json exists. + // This read cannot retire a live owner or accept a newer generation. + try { + // Warm turns advance the attempt lease while the physical runner keeps + // its initial PRP lease identity. Verify that original lease belongs to + // this exact owner lineage instead of requiring the latest attempt id. + const [physicalLease] = await input.db.select({ + providerLeaseId: environmentLeases.providerLeaseId, + metadata: environmentLeases.metadata, + }).from(environmentLeases).where(and( + eq(environmentLeases.id, input.identity.environmentLeaseId), + eq(environmentLeases.companyId, input.execution.binding.companyId), + eq(environmentLeases.environmentId, target.environmentId), + )).limit(1); + const leaseMetadata = record(physicalLease?.metadata); + const leaseOwner = record(leaseMetadata.computerOwner); + if (physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId || + leaseMetadata.agentId !== input.execution.binding.agentId || + leaseOwner.computerId !== priorWorkspace.computerOwner.computerId || + leaseOwner.ownerId !== priorWorkspace.computerOwner.ownerId || + !Number.isSafeInteger(leaseOwner.generation) || Number(leaseOwner.generation) < 1 || + Number(leaseOwner.generation) > priorWorkspace.computerOwner.generation) return "scope_mismatch"; + if (await computerService(input.db).isRetired({ + companyId: input.execution.binding.companyId, + environmentId: target.environmentId, + agentId: input.execution.binding.agentId, + runId: input.identity.runId, + owner: priorWorkspace.computerOwner, + })) return "verified"; + } catch { + return "unavailable"; + } + } return "terminal_state_indeterminate"; } catch { return "authority_indeterminate"; @@ -4896,6 +4942,7 @@ async function migrateRunnerdStateRootForExecution(input: { allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", + runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" && @@ -4940,6 +4987,7 @@ async function migrateRunnerdStateRootForExecution(input: { allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", + runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" &&