diff --git a/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md b/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md index 027e3595e38..bf02fb8396e 100644 --- a/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md +++ b/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md @@ -174,6 +174,13 @@ event acknowledged and no command in progress. Core then sends one ordinary and `runner` must return the original result and must not start a second Runner. Without `runner` on `resume`, keep the claimant handover above. +Core prepares the claimant run before this `resume`: its `prepare` has +`priorIdentity` set to the released source, and the released task is not yet +claimed. Core also takes this path when it recovers a released task after a +Core restart, if it recorded the build the task was released on and the +session's process configuration has not changed. A provider must then accept +that `prepare` without state it kept in memory before the restart. + While a released task can still be resumed, its `status` must report `executionStopped: false`, even when its Runner process is stopped. Core replaces a reused task whose `status` reports a stop, and then cannot resume diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 49f4e4ba43a..719559c3e2d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -12232,7 +12232,7 @@ async function executePaperclipNativeSessionAttempt( ); let existingWarmSession: NativeSession | undefined; let reusingReleasedRunner = false; - let relaunchReleased: { owner: ReturnType; source: DurableRecoveryIdentity; + let relaunchReleased: { owner?: ReturnType; source: DurableRecoveryIdentity; runner: { version: string; digest: string }; published?: true } | undefined; let managedCredentialSession: NativeSession | undefined; let githubAccess: NativeGitHubAccess | undefined; @@ -12560,24 +12560,44 @@ async function executePaperclipNativeSessionAttempt( pluginTaskDescription, retainedPluginTaskOwner: existingWarmSession ? pluginTaskSessionOwners.get(existingWarmSession) : relaunchReleased?.owner, relaunchReleased: relaunchReleased ? { source: relaunchReleased.source, runner: relaunchReleased.runner } : undefined, + onPluginTaskOwner: owner => { if (relaunchReleased && !relaunchReleased.owner) relaunchReleased.owner = owner; }, beginRetainedRunAttachment: existingWarmSession && reusingReleasedRunner ? beginRetainedRunAttachment(existingWarmSession) : undefined, durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, - remoteWarmRecovery: remoteWarmRecovery ? { - ...remoteWarmRecovery, + remoteWarmRecovery: remoteWarmRecovery ? (recovery => ({ + ...recovery, revalidate: async () => { - if (remoteWarmRecoveryPending) await remoteWarmRecovery.revalidate(leaseOwner); + if (remoteWarmRecoveryPending) await recovery.revalidate(leaseOwner); }, - } : undefined, + }))(remoteWarmRecovery) : undefined, trace, toolTrace, }); + const releasedRecovery = remoteWarmRecovery; + if (releasedRecovery?.releasedSource && releasedRecovery.releasedSourceRunner && releasedRecovery.reuseProcess + && !releasedRecovery.portableSourceTaskId && pluginTaskDescription?.resumeIdentityVersion === 1 && warmConfigDigest !== null) { + const checkpoint = loadWarmNativeCheckpoint(input.execution, warmConfigDigest, "remote"); + if (checkpoint) { + if (executionContext) await executionContext.wait(() => releasedRecovery.revalidate(leaseOwner)); + else await releasedRecovery.revalidate(leaseOwner); + relaunchReleased = { source: releasedRecovery.releasedSource, runner: releasedRecovery.releasedSourceRunner }; + persistedWarmSession = checkpoint; + remoteWarmRecovery = undefined; + remoteWarmRecoveryPending = false; + await emitStartupReuse(input.onEvent, "runner", { state: "fresh", reason: "replacement" }); + executionContext?.assertOpen(); + await emitStartupReuse(input.onEvent, "provider_session", { state: "reused", reason: "verified_reuse" }); + trace.setBudgetStart("warm"); + if (executionContext) await finishStartupContextForFallback(); + } + } let runnerdBackend = input.useRunnerd && input.backend === undefined ? await timeController("backend_setup", () => executionContext ? executionContext.wait(createExecutionBackend) : createExecutionBackend()) : null; if (remoteWarmRecovery) { - if (executionContext) await executionContext.wait(() => remoteWarmRecovery.revalidate(leaseOwner)); - else await remoteWarmRecovery.revalidate(leaseOwner); + const recovery = remoteWarmRecovery; + if (executionContext) await executionContext.wait(() => recovery.revalidate(leaseOwner)); + else await recovery.revalidate(leaseOwner); const recovered = await runnerdBackend?.recoverSession?.(remoteWarmRecovery.checkpoint, { signal: executionContext?.signal ?? new AbortController().signal, }); @@ -13109,7 +13129,7 @@ async function executePaperclipNativeSessionAttempt( try { result = await trace.run(runnerSessionStartupScope, runExecution); } catch (error) { - if (relaunchReleased && !relaunchReleased.published && !relaunchReleased.owner.relaunchMayHaveStarted()) { + if (relaunchReleased && !relaunchReleased.published && !relaunchReleased.owner?.relaunchMayHaveStarted()) { revertUnattachedRelaunchRotation(scopedRunnerdStateRoot(input.execution), relaunchReleased.source); } if (!await continueAfterUnavailableReuse(error)) throw error; @@ -16400,6 +16420,7 @@ export async function createRunnerdBackend(input: { remoteWarmRecovery?: Omit & { revalidate: () => Promise }; retainedPluginTaskOwner?: ReturnType; relaunchReleased?: { source: DurableRecoveryIdentity; runner: { version: string; digest: string } }; + onPluginTaskOwner?: (owner: ReturnType) => void; beginRetainedRunAttachment?: () => void; pluginTaskDescription?: PluginTaskDescription; onSpawn?: (meta: { @@ -16874,7 +16895,8 @@ async function createRunnerdBackendWithinSessionClaim( } let priorDescription = input.pluginTaskDescription?.taskId === remoteTarget.sandboxLeaseAcquisition?.providerLeaseId ? input.pluginTaskDescription : undefined; - if (remoteTarget.sandboxLeaseAcquisition?.outcome === "resumed" && !durableBinding && !input.retainedPluginTaskOwner && !input.remoteWarmRecovery) { + if (remoteTarget.sandboxLeaseAcquisition?.outcome === "resumed" && !durableBinding && !input.retainedPluginTaskOwner && !input.remoteWarmRecovery + && !input.relaunchReleased) { // A new logical session (including fresh retries) cannot inherit the old process authority. await pluginTask({kind:"stop"}); const stopped=environmentTaskResultSchema.parse(await pluginTask({kind:"status"})); @@ -16901,7 +16923,7 @@ async function createRunnerdBackendWithinSessionClaim( turnId: input.remoteWarmRecovery.identity.turnId, itemId: input.remoteWarmRecovery.identity.itemId, } : undefined; - const resumeFrom = !retainedOwner && !adopting && prior && prior.environmentLeaseId !== remoteTarget.leaseId + const resumeFrom = !retainedOwner && !adopting && !input.relaunchReleased && prior && prior.environmentLeaseId !== remoteTarget.leaseId ? await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, leaseId: remoteTarget.leaseId!, priorIdentity: prior }) : undefined; const credentials = await preparePluginRunnerCredentials({ @@ -16986,6 +17008,7 @@ async function createRunnerdBackendWithinSessionClaim( if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable"); const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput); if (retainedOwner) retainedOwner.rebind(ownerInput); + input.onPluginTaskOwner?.(owner); return { provider, description, prior, adoptionIdentity, adopting, owner }; } if (pluginTask && remoteTarget?.transport === "sandbox" && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8") { diff --git a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts index 1ce8a9f4f9d..4600c8aaa75 100644 --- a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts +++ b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts @@ -68,6 +68,7 @@ describeEmbeddedPostgres("plugin Runner release/resume through the Core executio nativeIt.for([...(["codex", "claude"] as const).flatMap(harness => (["cached", "adopted", "adopted restart"] as const).map(mode => [harness, mode] as const)), ["codex", "adopted cancellation"] as const, ["codex", "adopted legacy cancellation"] as const, ["codex", "cached cancellation"] as const, ["codex", "cached held claim"] as const, ["claude", "cached held claim"] as const, ["codex", "cached relaunch"] as const, ["claude", "cached relaunch"] as const, ["codex", "adopted relaunch"] as const, + ["codex", "adopted restart relaunch"] as const, ["claude", "adopted restart relaunch"] as const, ["codex", "cached managed GitHub"] as const, ["codex", "adopted managed GitHub"] as const, ["codex", "adopted restart managed GitHub"] as const])("completes %s runs through a retained %s process", { timeout: 60_000 }, async ([harness, mode], context) => { if (harness === "claude" && !existsSync(fakeAcpxSidecar)) context.skip(); const thisCase: { teardown?: () => Promise } = {}; @@ -263,7 +264,7 @@ describeEmbeddedPostgres("plugin Runner release/resume through the Core executio return { kind: "resume", taskId, outcome: "resumed", runner: artifact, archivedState }; } if (operation.kind === "resume") { - expect(relaunch && !adopted).toBe(false); + expect(relaunch).toBe(false); const retainedProviderCount = await providerStarts(); const reply = await host.request({ op: "claim", id: `claim_${sha(operation.bootstrapTicket)}`, parkId: parkId(operation.identity), bootstrapTicket: operation.bootstrapTicket }); if (reply.status !== "claimed") throw new Error("Fixture native claim refused"); @@ -386,19 +387,21 @@ describeEmbeddedPostgres("plugin Runner release/resume through the Core executio expect(releases[1]!.environmentLeaseId).toBe(leaseA); expect(submissions).toBe(1); if (relaunch) { expect(handle!.child.pid).not.toBe(pid); - const relaunched = adopted ? releases[1]! : releases[0]!; - expect(relaunches).toEqual([{ source: relaunched, runId: adopted ? runC : runB, archivedRunId: relaunched.runId }]); + const relaunched = adopted ? [[releases[0]!, runB], [releases[1]!, runC!]] as const : [[releases[0]!, runB]] as const; + expect(relaunches).toEqual(relaunched.map(([source, next]) => ({ source, runId: next, archivedRunId: source.runId }))); expect((await state()).runId).toBe(releases.at(-1)!.runId); - expect(JSON.parse(await readFile(join(runnerRoot, `archived-${relaunched.runId}.json`), "utf8"))).toMatchObject(relaunched); + for (const [source] of relaunched) { + expect(JSON.parse(await readFile(join(runnerRoot, `archived-${source.runId}.json`), "utf8"))).toMatchObject(source); + } } else expect(handle!.child.pid).toBe(pid); expect([...new Set(preparedBrokers.map(prepared => prepared.runId))]).toEqual(releases.map(release => release.runId)); const methods = (await readFile(calls, "utf8")).trim().split(/\r?\n/); if (harness === "codex") { - expect(methods.filter(method => method === "process-start")).toHaveLength(relaunch ? 2 : 1); + expect(methods.filter(method => method === "process-start")).toHaveLength(1 + relaunches.length); expect(methods.filter(method => method === "turn/start")).toHaveLength(adopted ? 3 : 2); } else { const journal = methods.map(line => JSON.parse(line)); - expect(journal.filter(entry => entry.processStart)).toHaveLength(adopted || relaunch ? 4 : 3); + expect(journal.filter(entry => entry.processStart)).toHaveLength((adopted ? 4 : 3) + relaunches.length); if (relaunch) { const attach = journal.findIndex(entry => entry.command === "run.attach" && entry.params?.runId === runB); const turn = journal.findIndex((entry, index) => index > attach && entry.command === "turn.start"); diff --git a/server/src/services/native-runtime/plugin-runner-task.test.ts b/server/src/services/native-runtime/plugin-runner-task.test.ts index 59257116a55..2f8bb99a885 100644 --- a/server/src/services/native-runtime/plugin-runner-task.test.ts +++ b/server/src/services/native-runtime/plugin-runner-task.test.ts @@ -813,15 +813,17 @@ describe("plugin Runner relaunch as the claimant run", () => { const spec = launchSpec(); return { ...spec, nativeRunner: { ...spec.nativeRunner!, identity: next, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } } }; }; - async function relaunchOwner(resume: (operation: any) => unknown, options: { resumeIdentityVersion?: 1; onResumed?: () => Promise } = { resumeIdentityVersion: 1 }) { + async function relaunchOwner(resume: (operation: any) => unknown, + options: { resumeIdentityVersion?: 1; onResumed?: () => Promise; fresh?: true } = { resumeIdentityVersion: 1 }) { const { task } = provider(); const perform = task.getMockImplementation()!; task.mockImplementation(async operation => operation.kind === "resume" ? resume(operation) : perform(operation)); const close = vi.fn(async () => {}); ingress.connect.mockReset().mockReturnValue({ ready: Promise.resolve(), quiesce: vi.fn(), close }); - const owner = createPluginRunnerTaskOwner({ ...base, task, adoptIdentity: source, authorityRunId: source.runId }); - owner.rebind({ ...base, task, authorityRunId: next.runId, resumeRelaunchedRunnerVersion: 1, - resumeIdentityVersion: options.resumeIdentityVersion, relaunch: { source, runner: parked }, onResumed: options.onResumed }); + const relaunchInput = { ...base, task, authorityRunId: next.runId, resumeRelaunchedRunnerVersion: 1 as const, + resumeIdentityVersion: options.resumeIdentityVersion, relaunch: { source, runner: parked }, onResumed: options.onResumed }; + const owner = createPluginRunnerTaskOwner(options.fresh ? relaunchInput : { ...base, task, adoptIdentity: source, authorityRunId: source.runId }); + if (!options.fresh) owner.rebind(relaunchInput); await prepare(owner, { ...requirements, identity: next }); const authority = { acceptRelaunchedRunner: vi.fn() }; const registration = await owner.registration(authority as never, next.runId, undefined, next); @@ -855,6 +857,30 @@ describe("plugin Runner relaunch as the claimant run", () => { } finally { await registration.release("detach"); } }); + it("relaunches through a new owner, as after a Core restart, without submitting or adopting", async () => { + const { owner, task, authority, registration } = await relaunchOwner( + () => ({ kind: "resume", taskId: "task", outcome: "resumed", runner: build, archivedState: settledCut }), + { resumeIdentityVersion: 1, fresh: true }); + expect(task.mock.calls.find(([operation]) => operation.kind === "prepare")?.[0]).toMatchObject({ priorIdentity: source }); + expect(owner.driver.relaunchReleasedRunner).toEqual(source); + expect(owner.driver.adoptExistingRunner).toBeUndefined(); + owner.launcher(relaunchSpec()); + registration.activate(); + try { + await registration.ready(); + expect(task.mock.calls.filter(([operation]) => operation.kind === "resume").map(([operation]) => operation.runner)) + .toEqual([{ runId: next.runId, turnId: next.turnId, itemId: next.itemId }]); + expect(task.mock.calls.some(([operation]) => operation.kind === "submit")).toBe(false); + expect(authority.acceptRelaunchedRunner).toHaveBeenCalledExactlyOnceWith("synthetic-ticket", build, parked); + } finally { await registration.release("detach"); } + }); + + it("treats a new owner as a new task when the provider does not advertise relaunch", async () => { + const { owner, task } = await relaunchOwner(() => { throw new Error("unexpected resume"); }, { fresh: true }); + expect(owner.driver.relaunchReleasedRunner).toBeUndefined(); + expect(task.mock.calls.find(([operation]) => operation.kind === "prepare")?.[0]).not.toHaveProperty("priorIdentity"); + }); + it("retries a lost reply with the same ticket and identity", async () => { let attempts = 0; const { task, registration, owner } = await relaunchOwner(() => { diff --git a/server/src/services/native-runtime/plugin-runner-task.ts b/server/src/services/native-runtime/plugin-runner-task.ts index 53022e2f414..fe2fb1fe526 100644 --- a/server/src/services/native-runtime/plugin-runner-task.ts +++ b/server/src/services/native-runtime/plugin-runner-task.ts @@ -76,12 +76,13 @@ export function createPluginRunnerTaskOwner(input: { let relaunchProgress: "none" | "sent" | "refused" = "none"; let activeOutbound: ReturnType | null = null; let preparedLaunchId: string | undefined; - let preparedIdentity: DurableRecoveryIdentity | undefined = input.adoptIdentity; - let launched = !!input.adoptIdentity; - let launchIdentity = input.adoptIdentity; + const relaunchedTask = !input.adoptIdentity && input.resumeIdentityVersion === 1 ? input.relaunch?.source : undefined; + let preparedIdentity: DurableRecoveryIdentity | undefined = input.adoptIdentity ?? relaunchedTask; + let launched = !!(input.adoptIdentity ?? relaunchedTask); + let launchIdentity = input.adoptIdentity ?? relaunchedTask; let stopRequested = false; let released = false; - let submitted = !!input.adoptIdentity; + let submitted = launched; let detached = false; let adoptionVerified = false; let markReady!: () => void; @@ -102,7 +103,7 @@ export function createPluginRunnerTaskOwner(input: { if (input.adoptIdentity) markReady(); const configuredEnvironmentFields = (launchesRunner: boolean) => input.configuredEnvironmentVersion === 1 && launchesRunner ? { configuredEnvironment: { ...input.configuredEnvironment } } : {}; - let configuredEnvironmentDelivered: boolean | undefined = input.adoptIdentity ? undefined : false; + let configuredEnvironmentDelivered: boolean | undefined = launched ? undefined : false; const call = async (operation: PluginEnvironmentTaskOperation) => { const phase = ({ prepare: "execution.plugin_prepare", check_project_auth: "execution.plugin_project_auth", submit: "execution.plugin_submit", connection: "execution.plugin_connection" } as const)[operation.kind as "prepare" | "check_project_auth" | "submit" | "connection"];