From d3fc9db89e8b8c43abbdc99a27686ee1d3630a2c Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 12:13:15 -0500 Subject: [PATCH] fix: verify original workspace lineage after warm retirement Co-Authored-By: Paperclip --- .../native-session-executor.test.ts | 34 +++++++++++---- .../native-runtime/native-session-executor.ts | 41 ++++++++++++++++--- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 0ac1c022ca..f0c8933da5 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -13457,6 +13457,13 @@ describe("runnerd provider runtime wiring", () => { it.each([ { condition: "retired", expected: "verified" }, { condition: "workspace-coordinate", expected: "verified" }, + { condition: "warm-origin", expected: "verified" }, + { condition: "origin-other-session", expected: "scope_mismatch" }, + { condition: "origin-other-owner", expected: "scope_mismatch" }, + { condition: "origin-newer-generation", expected: "scope_mismatch" }, + { condition: "origin-active", expected: "scope_mismatch" }, + { condition: "origin-ambiguous", expected: "scope_mismatch" }, + { condition: "origin-wrong-lease-run", expected: "scope_mismatch" }, { condition: "wrong-workspace-run", expected: "scope_mismatch" }, { condition: "uncertain", expected: "terminal_state_indeterminate" }, { condition: "stale-generation", expected: "terminal_state_indeterminate" }, @@ -13467,20 +13474,28 @@ describe("runnerd provider runtime wiring", () => { { condition: "corrupt-local", expected: "terminal_state_indeterminate" }, ])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => { const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-")); - const prior = execution; - const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } }; + const originCase = condition === "warm-origin" || condition.startsWith("origin-"); + const prior = originCase ? { ...execution, binding: { ...execution.binding, executionWorkspaceId: execution.binding.runId } } : execution; + const current = { ...prior, binding: { ...prior.binding, runId: "after-idle", ...(originCase ? { executionWorkspaceId: "after-idle" } : {}) } }; const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 }; const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId, remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation, listenerPort: owner.listenerPort, placementId: "placement" }; let leaseQuery: SQL | undefined; + const originExecution = { ...prior, binding: { ...prior.binding, runId: "origin-workspace", executionWorkspaceId: "origin-workspace" }, + session: { ...prior.session, normalizedSessionId: condition === "origin-other-session" ? "other-session" : prior.session.normalizedSessionId } }; + const originDescriptor = { ...descriptor, leaseId: "origin-lease", + providerLeaseId: condition === "origin-other-owner" ? "other-owner" : owner.ownerId, + ownerGeneration: condition === "origin-newer-generation" ? 3 : 1 }; + const originRow = { status: condition === "origin-active" ? "running" : "succeeded", + runnerProfileJson: { nativeExecutionInput: originExecution, nativeComputerWorkspace: originDescriptor } }; const workspaceCoordinate = ["workspace-coordinate", "wrong-workspace-run"].includes(condition); - const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async () => + const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async (limit: number) => table === environmentLeases ? condition === "wrong-lease" ? [] : [{ providerLeaseId: owner.ownerId, - heartbeatRunId: condition === "wrong-workspace-run" ? "unrelated-run" : prior.binding.runId, + heartbeatRunId: condition === "wrong-workspace-run" || condition === "origin-wrong-lease-run" ? "unrelated-run" : originCase ? "origin-workspace" : prior.binding.runId, metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } }, - }] : [{ status: condition === "active-run" ? "running" : "succeeded", + }] : limit === 2 ? (originCase ? condition === "origin-ambiguous" ? [originRow, originRow] : [originRow] : []) : [{ status: condition === "active-run" ? "running" : "succeeded", runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }], }); } }) }) } as unknown as Db; computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => { @@ -13489,7 +13504,7 @@ describe("runnerd provider runtime wiring", () => { }); const input = { db, root, execution: current, identity: { runId: prior.binding.runId, normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner", - environmentLeaseId: workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" }, + environmentLeaseId: originCase ? "origin-workspace" : workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" }, allowVerifiedBackup: false, remoteRunnerState: true, runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment", remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 }, @@ -13509,12 +13524,17 @@ describe("runnerd provider runtime wiring", () => { expect(computerRetirement.isRetired).not.toHaveBeenCalled(); } expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected); - if (["retired", "workspace-coordinate", "uncertain", "stale-generation", "unavailable"].includes(condition)) { + if (["retired", "workspace-coordinate", "warm-origin", "uncertain", "stale-generation", "unavailable"].includes(condition)) { expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({ companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId, runId: prior.binding.runId, owner, }); } else expect(computerRetirement.isRetired).not.toHaveBeenCalled(); + if (condition === "warm-origin") { + const query = new PgDialect().sqlToQuery(leaseQuery!); + expect(query.params).toContain("origin-lease"); + expect(query.params).not.toContain("origin-workspace"); + } if (workspaceCoordinate) { const query = new PgDialect().sqlToQuery(leaseQuery!); expect(query.params).toContain(descriptor.leaseId); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b4c7c07d1d..55ee4680f1 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4389,11 +4389,40 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { // environmentLeaseId. Computer placement records the actual lease. Only // this exact prior workspace coordinate may resolve through that record. // Explicit physical lease identities still preserve their warm lineage. - const workspaceCoordinate = input.identity.environmentLeaseId === - priorExecution.binding.executionWorkspaceId; - const physicalLeaseId = workspaceCoordinate - ? priorWorkspace.leaseId - : input.identity.environmentLeaseId; + let workspaceRunId: string | null = input.identity.environmentLeaseId === + priorExecution.binding.executionWorkspaceId ? input.identity.runId : null; + let physicalLeaseId = workspaceRunId ? priorWorkspace.leaseId : input.identity.environmentLeaseId; + if (!workspaceRunId) { + // Warm attachment advances runId while retaining the process's original + // workspace coordinate. Resolve that recorded origin, never an arbitrary + // run ID, before checking the physical lease and latest retirement. + const origins = await input.db.select({ + status: heartbeatRuns.status, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.execution.binding.companyId), + eq(heartbeatRuns.agentId, input.execution.binding.agentId), + eq(sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->>'executionWorkspaceId'`, + input.identity.environmentLeaseId), + )).limit(2); + if (origins.length > 1) return "scope_mismatch"; + if (origins.length === 1) { + const origin = origins[0]!; + const originProfile = record(origin.runnerProfileJson); + const originExecution = parseNativeExecutionInput(originProfile.nativeExecutionInput); + const originWorkspace = readNativeComputerWorkspaceReference(originProfile.nativeComputerWorkspace); + if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(origin.status) || + originExecution.binding.executionWorkspaceId !== input.identity.environmentLeaseId || + nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(priorExecution) || + !originWorkspace || originWorkspace.remoteCwd !== priorWorkspace.remoteCwd || + record(originProfile.nativeComputerWorkspace).placementId !== record(descriptor).placementId || + originWorkspace.computerOwner.computerId !== priorWorkspace.computerOwner.computerId || + originWorkspace.computerOwner.ownerId !== priorWorkspace.computerOwner.ownerId || + originWorkspace.computerOwner.generation > priorWorkspace.computerOwner.generation) return "scope_mismatch"; + workspaceRunId = originExecution.binding.runId; + physicalLeaseId = originWorkspace.leaseId; + } + } const [physicalLease] = await input.db.select({ providerLeaseId: environmentLeases.providerLeaseId, heartbeatRunId: environmentLeases.heartbeatRunId, @@ -4405,7 +4434,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: { )).limit(1); const leaseMetadata = record(physicalLease?.metadata); const leaseOwner = record(leaseMetadata.computerOwner); - if ((workspaceCoordinate && physicalLease?.heartbeatRunId !== input.identity.runId) || + if ((workspaceRunId && physicalLease?.heartbeatRunId !== workspaceRunId) || physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId || leaseMetadata.agentId !== input.execution.binding.agentId || leaseOwner.computerId !== priorWorkspace.computerOwner.computerId ||