diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index 527d18063aa..d377b91632c 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -12,8 +12,8 @@ import os from "node:os"; import { currentProcessSpaceHash, UNKNOWN_PROCESS_SPACE } from "./controller-process-identity.js"; import { proveOwnerGone } from "./owner-proof.js"; -export const PROCESS_START_REQUESTED = "native.process_start_requested"; -export const PROCESS_IDENTITY_RECORDED = "native.process_identity_recorded"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-process-start-events.js"; +export { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED }; /** Server-authored: the runner-state fence refused this run before it asked for * a runner, because a prior run's saved state could not be proven finished. */ export const PRIOR_OWNER_FENCE_REFUSED = "native.prior_owner_fence_refused"; diff --git a/server/src/services/native-process-start-events.ts b/server/src/services/native-process-start-events.ts new file mode 100644 index 00000000000..8167671ea55 --- /dev/null +++ b/server/src/services/native-process-start-events.ts @@ -0,0 +1,5 @@ +export const PROCESS_START_REQUESTED = "native.process_start_requested"; +export const PROCESS_IDENTITY_RECORDED = "native.process_identity_recorded"; + +/** Server-authored events (no source event ID) that prove a run started its own process. */ +export const OWN_PROCESS_START_EVENT_TYPES = [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED] as const; diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 6cf7f9b25b5..8e43a723ef6 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -433,6 +433,27 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]); }); + it("keeps a warm follow-up that inherited its predecessor's process identity on the same run until the outgoing owner's deadline", async () => { + const fixture = await seed(), runId = randomUUID(); + const scope = { ...fixture.scope, runId }; + const deadline = new Date(Date.now() + 60_000); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId, + nativeSessionId: scope.normalizedSessionId, status: "running", runtimeMode: "native", + processPid: 609, processGroupId: 609, processStartedAt: new Date(Date.now() - 120_000), + runnerProfileJson: { nativeExecutionInput: { binding: scope }, inheritedProcessIdentityRunId: fixture.scope.runId } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId)); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed" }); + await deferNativeControllerRecovery(db, { ...scope, retryAt: deadline }); + const [finalization] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(finalization).toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null, + nextAttemptAt: deadline, failureCode: NATIVE_RETAINED_OWNER_WAIT }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null }); + expect(run.runnerProfileJson).toMatchObject({ inheritedProcessIdentityRunId: fixture.scope.runId }); + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]); + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline })).toHaveLength(1); + }); + it.each(["committed", "workspace_finalizing"])("defers a remote warm continuation while its %s predecessor has a finalization lease", async phase => { const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(); const previous = parseNativeExecutionInput({ @@ -935,14 +956,21 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d } }); - it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => { + it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop", "own-process", "inherited-then-own-process", "inherited-then-own-identity"])("does not publish an admission wait after authority changes: %s", async scenario => { const fixture = await seed(), runId = randomUUID(); const scope = { ...fixture.scope, runId }; + const ownProcess = scenario.startsWith("own-process") || scenario.startsWith("inherited-then-own"); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId, status: scenario === "stopped" ? "cancelled" : "running", runtimeMode: "native", resultJson: scenario === "operator-stop" ? { cancellation: { source: "operator", expected: true, initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() } } : null, - runnerProfileJson: { nativeExecutionInput: { binding: scope } } }); + ...(ownProcess ? { processPid: 4242, processGroupId: 4242, processStartedAt: new Date() } : {}), + runnerProfileJson: { nativeExecutionInput: { binding: scope }, + ...(scenario.startsWith("inherited-then-own") ? { inheritedProcessIdentityRunId: fixture.scope.runId } : {}) } }); + if (scenario.startsWith("inherited-then-own")) { + await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1, + eventType: scenario === "inherited-then-own-process" ? "native.process_start_requested" : "native.process_identity_recorded" }); + } await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId)); await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed", attempt: scenario === "provider-attempt" ? 1 : 0, @@ -953,6 +981,9 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d retryAt: new Date(Date.now() + 60_000) })) .rejects.toThrow("wait_authority_changed"); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("observed"); + if (ownProcess) { + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0].processPid).toBe(4242); + } }); it("reports bounded retained-pointer coverage without loading historical payloads", async () => { diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index 859c7c837b7..a5696d40f02 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -1,10 +1,11 @@ import { posix } from "node:path"; -import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; +import { and, eq, inArray, isNull, ne, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { captureLegacyDurablePrpCheckpoint, parseNativeExecutionInput, type DurablePrpCheckpoint, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { readRunCancellation } from "../run-cancellation.js"; +import { OWN_PROCESS_START_EVENT_TYPES } from "../native-process-start-events.js"; import { nativeSha256 } from "./canonical.js"; import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js"; import { insertMigratedNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; @@ -99,16 +100,27 @@ export async function deferNativeControllerRecovery(db: Db, input: { const cancellation = record(run?.resultJson?.nativeCancellation); const stop = readRunCancellation(run?.resultJson); const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`); + const hasProcessIdentity = Boolean(run?.processPid || run?.processGroupId || run?.processStartedAt); + const inheritedProcessIdentity = hasProcessIdentity + && typeof record(run?.runnerProfileJson).inheritedProcessIdentityRunId === "string" + && !(await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, input.companyId), eq(heartbeatRunEvents.runId, input.runId), + isNull(heartbeatRunEvents.sourceEventId), + inArray(heartbeatRunEvents.eventType, [...OWN_PROCESS_START_EVENT_TYPES]))).limit(1)).length; if (!run || run.runtimeMode !== "native" || run.status !== "running" || task?.executionRunId !== input.runId || !owner || owner.phase !== "observed" || owner.resultId || owner.attempt !== 0 || owner.controllerGeneration !== 0 || owner.leaseOwner !== (input.leaseOwner ?? null) || (input.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt <= new Date(clock.now))) - || run.processPid || run.processGroupId || run.processStartedAt || authority || progress.size + || (hasProcessIdentity && !inheritedProcessIdentity) || authority || progress.size || !record(run.runnerProfileJson).nativeExecutionInput || run.resultJson?.startupCancellation || (stop?.source === "operator" && stop.expected) || (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) { throw new Error("native_controller_recovery_wait_authority_changed"); } + if (inheritedProcessIdentity) { + await tx.update(heartbeatRuns).set({ processPid: null, processGroupId: null, processStartedAt: null, updatedAt: new Date() }) + .where(eq(heartbeatRuns.id, input.runId)); + } await notifyNativeFinalizationWork(tx, input); await tx.update(nativeRunFinalizations).set({ phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, diff --git a/server/src/services/run-cost-accounting.ts b/server/src/services/run-cost-accounting.ts index fcf5d0062cf..dc6a235d08a 100644 --- a/server/src/services/run-cost-accounting.ts +++ b/server/src/services/run-cost-accounting.ts @@ -13,6 +13,7 @@ import { settleChargedImageReceiptsInTransaction } from "./native-runtime/genera import type { BudgetServiceHooks } from "./budgets.js"; import { queueBudgetEvaluation } from "./budget-evaluation-work.js"; import { ensureRuntimeAccountingBaseline } from "./runtime-accounting-totals.js"; +import { OWN_PROCESS_START_EVENT_TYPES } from "./native-process-start-events.js"; import { logger } from "../middleware/logger.js"; import { promises as fs } from "node:fs"; import { priceAnthropicReceipt } from "./anthropic-pricing.js"; @@ -141,7 +142,7 @@ const inheritedProcessIdentityOnly = sql`${heartbeatRuns.runnerProfileJson}->>'i and not exists (select 1 from ${heartbeatRunEvents} own_process where own_process.company_id = ${heartbeatRuns.companyId} and own_process.run_id = ${heartbeatRuns.id} and own_process.source_event_id is null - and own_process.event_type in ('native.process_start_requested', 'native.process_identity_recorded'))`; + and own_process.event_type in (${sql.join(OWN_PROCESS_START_EVENT_TYPES.map(type => sql`${type}`), sql`, `)}))`; const nativeProviderWorkNeverStarted = sql`${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} in ('failed', 'cancelled', 'timed_out', 'interrupted')