diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index b8c646ce8ba..682d1f8138f 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -189,6 +189,9 @@ the process owner's listener and TLS settings while rotating the run identity. Idle retirement suspends and stops Runner, saves its exact checkpoint, writes credentials through the durable lease handoff, then completes the provider task. +Core permits read-only suspension proof for the exact saved identity of a +successful warm run after its reuse lease is retained or superseded. This does +not authorize new work or file access through that lease. Temporary credential homes are not needed after the originating run ends. Termination receipts cover all run leases that owned the retired physical task. A stopped task gets a new provider task ID for checkpoint resumption. Inspection diff --git a/server/src/__tests__/environment-task-runtime.test.ts b/server/src/__tests__/environment-task-runtime.test.ts index 3d5b0a7c1df..ad59145248d 100644 --- a/server/src/__tests__/environment-task-runtime.test.ts +++ b/server/src/__tests__/environment-task-runtime.test.ts @@ -134,6 +134,30 @@ describe("plugin-provided Runner execution", () => { expect(environmentTaskOperationSchema.safeParse({ ...operation, path: "/arbitrary" }).success).toBe(false); expect(environmentTaskOperationSchema.safeParse({ kind: "archive_state", priorIdentity: identity, archiveKey: "../escape", resumeFrom: "prior" }).success).toBe(false); }); + it.each(["retained", "expired"])("reads the exact successful warm owner's state during %s cleanup", async status => { + const value = row(); + value.lease.status = status; + value.run!.status = "succeeded"; + Object.assign(value.lease, {leasePolicy:"reuse_by_environment"}); + Object.assign(value.lease.metadata, {runnerTaskDispatched:true,runnerTaskLifecyclePolicy:{mode:"warm",idleTimeoutMs:120000},runnerTaskIdentity:{...submit.runner}}); + const identity = {runnerInstanceId:"runner",environmentLeaseId:leaseId,runId:"run",normalizedSessionId:"session",turnId:"turn",itemId:"item"}; + const operation = {kind:"read_state" as const,identity}; + const result = {kind:"state",taskId:"attempt-1",state:{schema:"paperclip.runner.durable.state.v1",...identity,lifecycle:"suspended"}}; + const workers = worker(result); + await expect(executeEnvironmentTask(database(value).db,workers as never,{companyId:"company",leaseId,operation})).resolves.toEqual(result); + expect(workers.call).toHaveBeenCalledOnce(); + for (const change of ["identity","policy","dispatch","lease","run"] as const) { + const changed = structuredClone(value); + if (change === "policy") Object.assign(changed.lease.metadata,{runnerTaskLifecyclePolicy:{mode:"per_turn",idleTimeoutMs:null}}); + if (change === "dispatch") Object.assign(changed.lease.metadata,{runnerTaskDispatched:false}); + if (change === "lease") changed.lease.status="released"; + if (change === "run") changed.run!.status="cancelled"; + const denied=worker(result); + await expect(executeEnvironmentTask(database(changed).db,denied as never,{companyId:"company",leaseId, + operation:change === "identity" ? {...operation,identity:{...identity,itemId:"other"}} : operation})).rejects.toThrow(); + expect(denied.call).not.toHaveBeenCalled(); + } + }); it("requires project validation for the exact configuration, provider, and projects before credential dispatch", async () => { const operation = { ...submit, preparedLaunchId: "configuration", accountIdentity: "account", credential: { type: "codex-auth-json" as const, authJson: '{"OPENAI_API_KEY":"never-leak"}' } }; diff --git a/server/src/__tests__/plugin-warm-tasks.integration.test.ts b/server/src/__tests__/plugin-warm-tasks.integration.test.ts index aee224def81..d6a9f166683 100644 --- a/server/src/__tests__/plugin-warm-tasks.integration.test.ts +++ b/server/src/__tests__/plugin-warm-tasks.integration.test.ts @@ -84,6 +84,19 @@ const support=await getEmbeddedPostgresTestSupport(); const [old]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,f.first.lease.id)); expect(old.status).toBe("expired"); }); + it("keeps successful warm state readable for retirement before and after lease handoff",async()=>{ + const f=await fixture(); + const read=()=>f.runtime().task({companyId:f.companyId,leaseId:f.first.lease.id,operation:{kind:"read_state",identity:f.identity}}); + await expect(read()).resolves.toMatchObject({kind:"state"}); + await f.nextRun();await f.acquire(); + await expect(read()).resolves.toMatchObject({kind:"state"}); + const [prior]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,f.first.lease.id)); + expect(prior.status).toBe("expired"); + await expect(f.runtime().task({companyId:f.companyId,leaseId:f.first.lease.id,operation:{kind:"describe"}})).rejects.toThrow("not active"); + await expect(f.runtime().task({companyId:f.companyId,leaseId:f.first.lease.id,operation:{kind:"read_state",identity:{...f.identity,itemId:"foreign"}}})).rejects.toThrow(); + expect(f.calls).not.toContain("stop"); + expect(f.calls.filter(call=>call==="submit")).toHaveLength(1); + }); it("resumes stopped checkpoints through a new physical task instead of reusing a tombstone",async()=>{ const f=await fixture();f.stopped();await f.nextRun(); const next=await f.acquire(); diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index ffdbf703123..e7089e243fd 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -46,9 +46,6 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage const activeProbe = lease.heartbeatRunId === null && lease.issueId === null && probe && typeof probe.runId === "string" && typeof probe.sessionId === "string" && typeof probe.expiresAtMs === "number" && probe.expiresAtMs > Date.now(); - if ((operation.kind === "read_file" || operation.kind === "describe" || operation.kind === "prepare" || operation.kind === "submit" || operation.kind === "connection" || (operation.kind === "read_state" && metadata.runnerTaskExecutionStopped !== true) || operation.kind === "archive_state" || operation.kind === "check_project_auth" || operation.kind === "copilot_metadata") && - (metadata.runnerTaskRetiring === true || !environment || (!activeProbe && run?.status !== "running") || lease.status !== "active" || (lease.expiresAt && lease.expiresAt.getTime() <= Date.now()))) throw new Error("Plugin-provided Runner execution lease is not active"); - if (operation.kind === "copilot_metadata" && !activeProbe) throw new Error("Copilot metadata requires a private setup lease"); const checkpointIdentity = operation.kind === "read_state" || operation.kind === "read_file" ? operation.identity : operation.kind === "archive_state" ? operation.priorIdentity : null; const runner = checkpointIdentity ? { runnerId: checkpointIdentity.runnerInstanceId, leaseId: checkpointIdentity.environmentLeaseId, @@ -70,6 +67,13 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage const ownCheckpoint = checkpointIdentity && !resumeFrom && (matches(checkpointIdentity,savedIdentity) || operation.kind !== "read_file" && matches(checkpointIdentity,priorSaved)); if (checkpointIdentity && !resumeFrom && !ownCheckpoint) throw new Error("Plugin Runner state identity mismatch"); + const retainedCheckpointRead = operation.kind === "read_state" && ownCheckpoint && + lease.leasePolicy === "reuse_by_environment" && ["retained", "expired"].includes(lease.status) && + run?.status === "succeeded" && metadata.runnerTaskDispatched === true && + (metadata.runnerTaskLifecyclePolicy as {mode?:unknown}|undefined)?.mode === "warm"; + if ((operation.kind === "read_file" || operation.kind === "describe" || operation.kind === "prepare" || operation.kind === "submit" || operation.kind === "connection" || (operation.kind === "read_state" && metadata.runnerTaskExecutionStopped !== true && !retainedCheckpointRead) || operation.kind === "archive_state" || operation.kind === "check_project_auth" || operation.kind === "copilot_metadata") && + (metadata.runnerTaskRetiring === true || !environment || (!activeProbe && run?.status !== "running") || lease.status !== "active" || (lease.expiresAt && lease.expiresAt.getTime() <= Date.now()))) throw new Error("Plugin-provided Runner execution lease is not active"); + if (operation.kind === "copilot_metadata" && !activeProbe) throw new Error("Copilot metadata requires a private setup lease"); const warmAttachment = !!prior && metadata.runnerTaskReused === true && (matches(prior, savedIdentity) || matches(prior, priorSaved) && operation.kind === "prepare" && matches(operation.requirements.identity, savedIdentity)) && runner?.runnerId === prior.runnerInstanceId && runner?.leaseId === prior.environmentLeaseId && runner?.sessionId === prior.normalizedSessionId;