From d6363b41d4502f255a28ff3bc7cff5a2fed811a6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 10:52:34 -0500 Subject: [PATCH 1/3] fix(runtime): verify retired computer owners after warm expiry Co-Authored-By: Paperclip --- .../native-session-executor.test.ts | 63 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 48 ++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index e03c1f8333..4bd3472ee3 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -28,6 +28,7 @@ import { inspect } from "node:util"; import { join } from "node:path"; import { heartbeatRuns, + environmentLeases, heartbeatRunEvents, issues, nativeRunFinalizations, @@ -82,6 +83,9 @@ vi.mock("node:fs", async (importOriginal) => { return { ...actual, readFileSync: vi.fn(actual.readFileSync) }; }); +const computerRetirement = vi.hoisted(() => ({ isRetired: vi.fn() })); +vi.mock("../../modules/computers/index.js", () => ({ computerService: () => computerRetirement })); + const githubAccess = vi.hoisted(() => ({ activate: vi.fn((_binding: { runId: string }) => vi.fn()), stop: vi.fn(async () => undefined), @@ -13438,6 +13442,65 @@ describe("runnerd provider runtime wiring", () => { } finally { await rm(root, { recursive: true, force: true }); } }); + it.each([ + { condition: "retired", expected: "verified" }, + { condition: "uncertain", expected: "terminal_state_indeterminate" }, + { condition: "stale-generation", expected: "terminal_state_indeterminate" }, + { condition: "wrong-placement", expected: "scope_mismatch" }, + { condition: "wrong-lease", expected: "scope_mismatch" }, + { condition: "active-run", expected: "active" }, + { condition: "unavailable", expected: "unavailable" }, + { condition: "corrupt-local", expected: "terminal_state_indeterminate" }, + ])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-")); + const prior = execution; + const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } }; + const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 }; + const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId, + remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation, + listenerPort: owner.listenerPort, placementId: "placement" }; + const db = { select: () => ({ from: (table: unknown) => ({ where: () => ({ limit: async () => + table === environmentLeases ? condition === "wrong-lease" ? [] : [{ + providerLeaseId: owner.ownerId, + metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } }, + }] : [{ status: condition === "active-run" ? "running" : "succeeded", + runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }], + }) }) }) } as unknown as Db; + computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => { + if (condition === "unavailable") throw new Error("ledger unavailable"); + return condition !== "uncertain" && requested.generation === (condition === "stale-generation" ? 3 : 2); + }); + const input = { db, root, execution: current, identity: { runId: prior.binding.runId, + normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner", + environmentLeaseId: condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" }, + allowVerifiedBackup: false, remoteRunnerState: true, + runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment", + remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 }, + fileAuthority: { placementId: condition === "wrong-placement" ? "other-placement" : "placement" } } as never }; + try { + if (condition === "corrupt-local") { + await mkdir(join(root, "runner")); + await writeFile(join(root, "runner", "runner-state.json"), "invalid state"); + } + if (condition === "retired") { + // Both live warm handoffs use their retained owner. After idle expiry, + // the next turn must instead prove the exact generation was retired. + for (let turn = 0; turn < 2; turn += 1) { + expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: true })) + .toBe("retained_warm_runner"); + } + expect(computerRetirement.isRetired).not.toHaveBeenCalled(); + } + expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected); + if (["retired", "uncertain", "stale-generation", "unavailable"].includes(condition)) { + expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({ + companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId, + runId: prior.binding.runId, owner, + }); + } else expect(computerRetirement.isRetired).not.toHaveBeenCalled(); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it.each([ { override: undefined, expected: "@openai/codex@0.160.0" }, { override: "@openai/codex@0.159.0", expected: "@openai/codex@0.159.0" }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c5bf41a377..d411b71e70 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,5 @@ +import { computerService } from "../../modules/computers/index.js"; +import { readNativeComputerWorkspaceReference } from "./native-workspace-sync.js"; import { adapterExecutionTargetIsCommandBacked, type AdapterComputerExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { configuredEnvironment } from "../../vendor/paperclip-runner/index.js"; import { agents } from "@paperclipai/db"; @@ -4295,6 +4297,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { allowVerifiedBackup: boolean; allowRetainedWarmRunner: boolean; remoteRunnerState?: boolean; + runnerExecutionTarget?: AdapterExecutionTarget | null; }): Promise { let priorRun: { status: string; @@ -4352,6 +4355,49 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { ) { return "retained_warm_runner"; } + const target = input.runnerExecutionTarget; + if (input.remoteRunnerState && directLifecycle === "absent" && + target?.kind === "remote" && target.transport === "computer" && target.environmentId) { + const descriptor = record(priorRun.runnerProfileJson).nativeComputerWorkspace; + const priorWorkspace = readNativeComputerWorkspaceReference(descriptor); + if (!priorWorkspace || priorWorkspace.computerOwner.computerId !== target.resourceAuthority.computerId || + priorWorkspace.remoteCwd !== target.remoteCwd || + record(descriptor).placementId !== target.fileAuthority.placementId) return "scope_mismatch"; + // Persistent runners keep their state remotely. Once the exact prior + // owner has completed retirement, its terminal run and fenced process + // are authoritative even though no local runner-state.json exists. + // This read cannot retire a live owner or accept a newer generation. + try { + // Warm turns advance the attempt lease while the physical runner keeps + // its initial PRP lease identity. Verify that original lease belongs to + // this exact owner lineage instead of requiring the latest attempt id. + const [physicalLease] = await input.db.select({ + providerLeaseId: environmentLeases.providerLeaseId, + metadata: environmentLeases.metadata, + }).from(environmentLeases).where(and( + eq(environmentLeases.id, input.identity.environmentLeaseId), + eq(environmentLeases.companyId, input.execution.binding.companyId), + eq(environmentLeases.environmentId, target.environmentId), + )).limit(1); + const leaseMetadata = record(physicalLease?.metadata); + const leaseOwner = record(leaseMetadata.computerOwner); + if (physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId || + leaseMetadata.agentId !== input.execution.binding.agentId || + leaseOwner.computerId !== priorWorkspace.computerOwner.computerId || + leaseOwner.ownerId !== priorWorkspace.computerOwner.ownerId || + !Number.isSafeInteger(leaseOwner.generation) || Number(leaseOwner.generation) < 1 || + Number(leaseOwner.generation) > priorWorkspace.computerOwner.generation) return "scope_mismatch"; + if (await computerService(input.db).isRetired({ + companyId: input.execution.binding.companyId, + environmentId: target.environmentId, + agentId: input.execution.binding.agentId, + runId: input.identity.runId, + owner: priorWorkspace.computerOwner, + })) return "verified"; + } catch { + return "unavailable"; + } + } return "terminal_state_indeterminate"; } catch { return "authority_indeterminate"; @@ -4896,6 +4942,7 @@ async function migrateRunnerdStateRootForExecution(input: { allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", + runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" && @@ -4940,6 +4987,7 @@ async function migrateRunnerdStateRootForExecution(input: { allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", + runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" && From e90fbf20a1d3c42d89b5cf3848c5df843eb646ad Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 10:53:51 -0500 Subject: [PATCH 2/3] fix(adapters): isolate computer legacy runtime assets by owner Co-Authored-By: Paperclip --- .../src/execution-target.test.ts | 36 +++++++++++++++++++ .../adapter-utils/src/execution-target.ts | 24 ++++++++++--- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/adapter-utils/src/execution-target.test.ts b/packages/adapter-utils/src/execution-target.test.ts index 46a4df7d6d..88919a99f3 100644 --- a/packages/adapter-utils/src/execution-target.test.ts +++ b/packages/adapter-utils/src/execution-target.test.ts @@ -5,6 +5,9 @@ import * as ssh from "./ssh.js"; import * as serverUtils from "./server-utils.js"; import { cleanupGitHubOperationLaunchers, + adapterExecutionTargetRuntimeRoot, + prepareAdapterExecutionTargetRuntime, + type AdapterComputerExecutionTarget, prepareGitHubOperationLaunchers, adapterExecutionTargetUsesManagedHome, ensureAdapterExecutionTargetRuntimeCommandInstalled, @@ -437,3 +440,36 @@ describe("GitHub launcher lifecycle", () => { expect(runner.execute).toHaveBeenCalledTimes(1); }); }); + + +describe("computer legacy runtime ownership", () => { + it.each(["claude", "codex"])("isolates %s assets and callback roots for owners sharing a checkout", async adapterKey => { + const execute = vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: 123, startedAt: "2026-10-10T00:00:00Z" })); + const target = (ownerId: string, generation = 1) => ({ + kind: "remote", transport: "computer", providerKey: "boat", environmentId: "environment", + leaseId: ownerId, remoteCwd: "/shared/project", listenerPort: 43127, + runner: { execute }, processRunner: { execute }, + resourceAuthority: { kind: "computer-owner", computerId: "computer", ownerId, generation }, + fileAuthority: { kind: "remote-persistent", placementId: "shared", root: "/shared/project", agentHome: `/homes/${ownerId}` }, + launch: vi.fn(), inspectProcess: vi.fn(), retainWarm: vi.fn(), retire: vi.fn(), + computerTool: { command: "cua-driver", args: ["mcp"] }, + } satisfies AdapterComputerExecutionTarget); + const prepare = (owner: AdapterComputerExecutionTarget) => prepareAdapterExecutionTargetRuntime({ + target: owner, runId: owner.leaseId!, adapterKey, workspaceLocalDir: "/unused-local-workspace", syncWorkspace: false, + }); + const first = await prepare(target("agent-one")); + const second = await prepare(target("agent-two")); + expect(first.workspaceRemoteDir).toBe("/shared/project"); + expect(second.workspaceRemoteDir).toBe(first.workspaceRemoteDir); + expect(first.runtimeRootDir).toBe(adapterExecutionTargetRuntimeRoot(target("agent-one"), adapterKey)); + expect(second.runtimeRootDir).not.toBe(first.runtimeRootDir); + expect(adapterExecutionTargetRuntimeRoot(target("agent-one", 2), adapterKey)).not.toBe(first.runtimeRootDir); + expect((await prepare(target("agent-one"))).runtimeRootDir).toBe(first.runtimeRootDir); + expect(first.runtimeRootDir).toMatch(new RegExp(`^/shared/project/\\.paperclip-runtime/${adapterKey}/owners/[a-f0-9]{64}$`)); + }); + + it("preserves the existing sandbox runtime root", () => { + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" } as const; + expect(adapterExecutionTargetRuntimeRoot(target, "claude")).toBe("/workspace/.paperclip-runtime/claude"); + }); +}); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index dbffa195aa..73595d5cfa 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -6,7 +6,7 @@ import { promisify } from "node:util"; import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { randomBytes, randomUUID } from "node:crypto"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; import { githubLauncherSource } from "./github-launcher.js"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { @@ -1442,6 +1442,20 @@ export function readAdapterExecutionTarget(input: { ); } +/** Keep mutable legacy assets separate for owners sharing the same checkout. */ +export function adapterExecutionTargetRuntimeRoot( + target: Extract, + adapterKey: string, + baseRoot = path.posix.join(target.remoteCwd, ".paperclip-runtime", adapterKey), +): string { + if (target.transport !== "computer") return baseRoot; + const owner = target.resourceAuthority; + const scope = createHash("sha256").update(JSON.stringify([ + owner.computerId, owner.ownerId, owner.generation, + ])).digest("hex"); + return path.posix.join(baseRoot, "owners", scope); +} + export async function prepareAdapterExecutionTargetRuntime(input: { runId: string; target: AdapterExecutionTarget | null | undefined; @@ -1535,7 +1549,9 @@ export async function prepareAdapterExecutionTargetRuntime(input: { adapterKey: input.adapterKey, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, - runtimeRootDir: input.runtimeRootDir, + runtimeRootDir: target.transport === "computer" + ? adapterExecutionTargetRuntimeRoot(target, input.adapterKey, input.runtimeRootDir) + : input.runtimeRootDir, syncWorkspace: input.syncWorkspace, workspaceInboundMode: input.workspaceInboundMode, workspaceDurableSeed: input.workspaceDurableSeed, @@ -2033,7 +2049,7 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { ? Math.trunc(input.timeoutSec * 1000) : target.timeoutMs ?? undefined; const bridgeRuntimeDir = path.posix.join( - input.runtimeRootDir?.trim() || path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey), + input.runtimeRootDir?.trim() || adapterExecutionTargetRuntimeRoot(target, input.adapterKey), "process-sessions", ); const sessionId = randomUUID(); @@ -4402,7 +4418,7 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { const runtimeRootDir = input.runtimeRootDir?.trim().length ? input.runtimeRootDir.trim() - : path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey); + : adapterExecutionTargetRuntimeRoot(target, input.adapterKey); const bridgeRuntimeDir = path.posix.join(runtimeRootDir, "paperclip-bridge"); const queueDir = path.posix.join(bridgeRuntimeDir, "queue"); const assetRemoteDir = path.posix.join(bridgeRuntimeDir, "server"); From 82880b10acef7ace137920429e7f6acef232b2bb Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 10:54:42 -0500 Subject: [PATCH 3/3] Keep legacy runtime acceptance coverage in stacked PR The combined feature branch retains the complete execution-target tests; this runtime branch preserves the production fix with its existing baseline suite. Co-Authored-By: Paperclip --- .../src/execution-target.test.ts | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/packages/adapter-utils/src/execution-target.test.ts b/packages/adapter-utils/src/execution-target.test.ts index 88919a99f3..46a4df7d6d 100644 --- a/packages/adapter-utils/src/execution-target.test.ts +++ b/packages/adapter-utils/src/execution-target.test.ts @@ -5,9 +5,6 @@ import * as ssh from "./ssh.js"; import * as serverUtils from "./server-utils.js"; import { cleanupGitHubOperationLaunchers, - adapterExecutionTargetRuntimeRoot, - prepareAdapterExecutionTargetRuntime, - type AdapterComputerExecutionTarget, prepareGitHubOperationLaunchers, adapterExecutionTargetUsesManagedHome, ensureAdapterExecutionTargetRuntimeCommandInstalled, @@ -440,36 +437,3 @@ describe("GitHub launcher lifecycle", () => { expect(runner.execute).toHaveBeenCalledTimes(1); }); }); - - -describe("computer legacy runtime ownership", () => { - it.each(["claude", "codex"])("isolates %s assets and callback roots for owners sharing a checkout", async adapterKey => { - const execute = vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: 123, startedAt: "2026-10-10T00:00:00Z" })); - const target = (ownerId: string, generation = 1) => ({ - kind: "remote", transport: "computer", providerKey: "boat", environmentId: "environment", - leaseId: ownerId, remoteCwd: "/shared/project", listenerPort: 43127, - runner: { execute }, processRunner: { execute }, - resourceAuthority: { kind: "computer-owner", computerId: "computer", ownerId, generation }, - fileAuthority: { kind: "remote-persistent", placementId: "shared", root: "/shared/project", agentHome: `/homes/${ownerId}` }, - launch: vi.fn(), inspectProcess: vi.fn(), retainWarm: vi.fn(), retire: vi.fn(), - computerTool: { command: "cua-driver", args: ["mcp"] }, - } satisfies AdapterComputerExecutionTarget); - const prepare = (owner: AdapterComputerExecutionTarget) => prepareAdapterExecutionTargetRuntime({ - target: owner, runId: owner.leaseId!, adapterKey, workspaceLocalDir: "/unused-local-workspace", syncWorkspace: false, - }); - const first = await prepare(target("agent-one")); - const second = await prepare(target("agent-two")); - expect(first.workspaceRemoteDir).toBe("/shared/project"); - expect(second.workspaceRemoteDir).toBe(first.workspaceRemoteDir); - expect(first.runtimeRootDir).toBe(adapterExecutionTargetRuntimeRoot(target("agent-one"), adapterKey)); - expect(second.runtimeRootDir).not.toBe(first.runtimeRootDir); - expect(adapterExecutionTargetRuntimeRoot(target("agent-one", 2), adapterKey)).not.toBe(first.runtimeRootDir); - expect((await prepare(target("agent-one"))).runtimeRootDir).toBe(first.runtimeRootDir); - expect(first.runtimeRootDir).toMatch(new RegExp(`^/shared/project/\\.paperclip-runtime/${adapterKey}/owners/[a-f0-9]{64}$`)); - }); - - it("preserves the existing sandbox runtime root", () => { - const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" } as const; - expect(adapterExecutionTargetRuntimeRoot(target, "claude")).toBe("/workspace/.paperclip-runtime/claude"); - }); -});