diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 2ceb6be9c74..61bffc56443 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2531,3 +2531,15 @@ restored checkpoint must match the normalized session before writing its local projection. This adds a scoped checkpoint read and local reconstruction only when warm admission encounters an external marker or an older projection; it adds no streaming checkpoint writes or recurring idle work. + +Ordinary remote `per_turn` continuations restore a valid external controller +marker from the referenced run's company/task-bound checkpoint before reading +its durable identity. The restored checkpoint must match the normalized session. +An existing prior-run projection also refreshes from that checkpoint, so a +rejected attempt cannot retain state that its active owner later advances. +Current-run full state stays owned by its controller. Legacy file-backed state +retains its existing admission path when no database checkpoint exists. +The existing prior-owner, suspended-state and backup checks still decide whether +the next turn may start. Missing checkpoints and ambiguous identity remain +blocked; restoration does not authorize a new provider session or change the +configured lifecycle policy. diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index 1d0cfb31992..d5892c37ec5 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -299,6 +299,16 @@ Policy: Task-session listing and execution-context reset require board agent management at their HTTP routes. These controls are separate from delegated managed AI-account authentication attempts under `model-accounts`. +Ordinary remote `per_turn` continuation restores an external controller marker +through the existing company/task-bound checkpoint store, with the expected +normalized session. Prior-owner, lifecycle and backup admission checks remain +authoritative. Prior-run projections refresh when that checkpoint exists; legacy +file-backed admission remains available when it does not. This changes no agent +grant or session-control endpoint. The +[executor controls](../../server/src/services/native-runtime/native-session-executor.test.ts) +provide source-level restoration and denial evidence; live provider continuity +is a separate verification step. + Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L6608) at the reviewed commit above. diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index e6bade34f1d..5d86501621a 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -74,6 +74,17 @@ mutations. Qualify a useful model task separately from saving the account. ### `session` +For a remote OpenCode agent configured with `per_turn`, complete a task, then +send two follow-ups without resetting its session. Verify that each continuation +restores its external controller checkpoint, retains provider context and saved +files, and finishes with no stranded task lock. A missing checkpoint, an active +prior owner or a mismatched session must still stop admission. The +rejected active-owner attempt must not prevent a later continuation from reading +the owner's newer completed checkpoint. The +[executor controls](../../server/src/services/native-runtime/native-session-executor.test.ts) +cover checkpoint restoration and its guards; real-provider acceptance remains +separate evidence. + After a completed warm turn, change an agent setting that invalidates its instruction copy, then send a follow-up. Verify the old runner suspends during preparation, saved files remain available, and the follow-up completes once. diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 97dd4685495..add9cfac21e 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11262,10 +11262,13 @@ describe("runnerd provider runtime wiring", () => { function priorNativeRunDb(priorExecution: NativeExecutionInput, status = "succeeded"): Db { return { - select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ - status, - runnerProfileJson: { nativeExecutionInput: priorExecution }, - }] }) }) }), + select: () => ({ from: (table: unknown) => { + const result = Promise.resolve(table === heartbeatRuns ? [{ + status, runnerProfileJson: { nativeExecutionInput: priorExecution }, + }] : []); + const query = { then: result.then.bind(result), where: () => query, limit: () => query }; + return query; + } }), } as unknown as Db; } @@ -13118,6 +13121,166 @@ describe("runnerd provider runtime wiring", () => { expect(state.createTransport).not.toHaveBeenCalled(); }); + it("refreshes a per-turn controller projection after rejecting an active prior owner", async () => { + const prior = parseNativeExecutionInput({ + ...scopedNativeExecution("per-turn-active-projection"), + schema: "paperclip.native-execution-input.v5", + executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(), + provider: { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "allow" }, + session: { ...execution.session, normalizedSessionId: "session-per-turn-active-projection", + driverKind: "opencode_server", lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + }); + const current = { ...prior, binding: { ...prior.binding, runId: `${prior.binding.runId}-next` } }; + const identity = { runId: prior.binding.runId, normalizedSessionId: prior.session.normalizedSessionId!, + runnerInstanceId: "runner-per-turn-active-projection", environmentLeaseId: "lease-per-turn-active-projection", + turnId: "turn-prior", itemId: "item-prior" }; + const root = await initializeScopedRunnerRoot(prior, identity.runnerInstanceId); + const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity, + expectedRunnerVersion: "per-turn-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + let saved: import("../../vendor/paperclip-runner/index.js").DurablePrpCheckpoint | undefined; + controller.setCheckpointPersistence(async checkpoint => { saved = checkpoint; }); + await controller.flushCheckpoint(); + await mkdir(join(root, "runner"), { recursive: true }); + await writeFile(join(root, "runner/runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready"))); + const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockImplementation(async (_db, binding, directory, sessionId) => { + expect(binding).toEqual({ companyId: prior.binding.companyId, issueId: prior.binding.issueId, runId: prior.binding.runId }); + expect(sessionId).toBe(prior.session.normalizedSessionId); + saved!.restoreSessionDirectory(directory); + return saved!; + }); + const remote = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner: { execute: vi.fn() } } as const; + state.createBackend.mockClear(); + state.createTransport.mockClear(); + try { + await expect(createRunnerdBackend({ db: priorNativeRunDb(prior, "running"), execution: current, + runnerInstanceId: "runner-next", runnerExecutionTarget: remote as never })) + .rejects.toThrow("runner_state_identity_mismatch: prior_owner_active"); + const projected = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); + expect(projected).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v1", commands: [] }); + expect(projected.runAttachTemplate).toBeNull(); + const template = { provider: prior.provider, normalizedSessionId: identity.normalizedSessionId }; + controller.persistRunAttachTemplate(template); + controller.queueCommand("run.prepare", template); + await controller.flushCheckpoint(); + await writeFile(join(root, "runner/runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended"))); + const admitted = new Error("per-turn resume reached tool authority"); + state.toolAuthorityDefinitions.mockRejectedValueOnce(admitted); + await expect(createRunnerdBackend({ db: priorNativeRunDb(prior), execution: current, + runnerInstanceId: "runner-next", runnerExecutionTarget: remote as never })).rejects.toBe(admitted); + const refreshed = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); + expect(refreshed.runAttachTemplate).toEqual(template); + expect(refreshed.commands).toEqual([expect.objectContaining({ type: "run.prepare", payload: template })]); + expect(restore).toHaveBeenCalledTimes(2); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + expect(existsSync(join(isolatedStateDirectory, "quarantine"))).toBe(false); + } finally { + state.toolAuthorityDefinitions.mockReset().mockResolvedValue([]); + restore.mockRestore(); + await controller.stop(); + } + }); + + it.each(["suspended", "backup", "codex", "file-projection", "legacy-file-no-checkpoint", "file-checkpoint-binding-error", + "missing-checkpoint", "checkpoint-binding-error", "foreign-task", "foreign-marker-session", + "missing-marker", "malformed-marker", "unknown-marker-schema", "active-owner", "foreign-prior-scope", "unsuspended-runner"] as const)( + "restores only an authorized externally persisted per-turn controller before ordinary resume: %s", async scenario => { + const prior = parseNativeExecutionInput({ + ...scopedNativeExecution("per-turn-external-resume"), + schema: "paperclip.native-execution-input.v5", + executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(), + provider: scenario === "codex" ? { kind: "codex", model: null, approvalPolicy: "never" } + : { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "allow" }, + session: { ...execution.session, normalizedSessionId: "session-per-turn-external-resume", + driverKind: scenario === "codex" ? "codex_app_server" : "opencode_server", + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + }); + const current = { ...prior, binding: { ...prior.binding, runId: `${prior.binding.runId}-next`, + issueId: scenario === "foreign-task" ? "another-task" : prior.binding.issueId } }; + const identity = { runId: prior.binding.runId, normalizedSessionId: prior.session.normalizedSessionId!, + runnerInstanceId: "runner-per-turn-external-resume", environmentLeaseId: "lease-per-turn-external-resume", + turnId: "turn-prior", itemId: "item-prior" }; + const root = await initializeScopedRunnerRoot(prior, identity.runnerInstanceId); + const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity, + expectedRunnerVersion: "per-turn-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + let saved: import("../../vendor/paperclip-runner/index.js").DurablePrpCheckpoint | undefined; + controller.setCheckpointPersistence(async checkpoint => { saved = checkpoint; }); + await controller.flushCheckpoint(); + expect(JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).schema) + .toBe("paperclip.runner.external-control-plane-state.v1"); + if (["file-projection", "legacy-file-no-checkpoint", "file-checkpoint-binding-error"].includes(scenario)) + saved!.restoreSessionDirectory(root); + if (scenario === "foreign-marker-session" || scenario === "unknown-marker-schema") { + const path = join(root, "control-plane/control-plane-state.json"); + const marker = JSON.parse(await readFile(path, "utf8")); + if (scenario === "foreign-marker-session") marker.identity.normalizedSessionId = "another-session"; + else marker.schema = "paperclip.runner.external-control-plane-state.v999"; + await writeFile(path, JSON.stringify(marker)); + } + if (scenario === "missing-marker") await rm(join(root, "control-plane/control-plane-state.json")); + if (scenario === "malformed-marker") await writeFile(join(root, "control-plane/control-plane-state.json"), "{"); + if (scenario === "backup") { + const backupRoot = join(root, "failover-backups/current"); + for (const directory of resolveNativeHarnessPersistenceProfile(prior).directories) + await mkdir(join(backupRoot, directory.name), { recursive: true }); + await writeFile(join(backupRoot, "runner/runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended"))); + await writeFile(join(backupRoot, "manifest.json"), JSON.stringify(buildNativeHarnessBackupManifest({ + backupRoot, execution: prior, runnerInstanceId: identity.runnerInstanceId, + providerSessionIdentity: { providerSessionId: "provider-per-turn-resume", providerBackendSessionId: null, + providerSessionIdentity: null }, sourceProviderLeaseId: "sandbox-per-turn-resume", + }))); + } else { + await mkdir(join(root, "runner"), { recursive: true }); + await writeFile(join(root, "runner/runner-state.json"), JSON.stringify(durableRunnerState(identity, + scenario === "unsuspended-runner" ? "ready" : "suspended"))); + } + const markerBefore = scenario === "missing-marker" ? null + : await readFile(join(root, "control-plane/control-plane-state.json"), "utf8"); + const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockImplementation(async (_db, binding, directory, sessionId) => { + expect(binding).toEqual({ companyId: prior.binding.companyId, issueId: current.binding.issueId, runId: prior.binding.runId }); + expect(sessionId).toBe(prior.session.normalizedSessionId); + if (["missing-checkpoint", "legacy-file-no-checkpoint"].includes(scenario) || binding.issueId !== prior.binding.issueId) return null; + if (["checkpoint-binding-error", "file-checkpoint-binding-error"].includes(scenario)) throw new Error("native_runner_checkpoint_binding_mismatch"); + saved!.restoreSessionDirectory(directory); + return saved!; + }); + state.createBackend.mockClear(); + state.createTransport.mockClear(); + const admitted = new Error("per-turn resume reached tool authority"); + state.toolAuthorityDefinitions.mockRejectedValueOnce(admitted); + try { + const persistedPrior = scenario === "foreign-prior-scope" + ? { ...prior, binding: { ...prior.binding, agentId: "another-agent" } } : prior; + const resume = createRunnerdBackend({ db: priorNativeRunDb(persistedPrior, scenario === "active-owner" ? "running" : "succeeded"), execution: current, + runnerInstanceId: "runner-next", runnerExecutionTarget: { kind: "remote", transport: "sandbox", + remoteCwd: "/workspace", runner: { execute: vi.fn() } } as never }); + if (["suspended", "backup", "codex", "file-projection", "legacy-file-no-checkpoint"].includes(scenario)) { + await expect(resume).rejects.toBe(admitted); + const restored = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); + expect(restored).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v1", identity }); + } else { + const reason = ["missing-checkpoint", "foreign-task"].includes(scenario) ? "native_runner_checkpoint_missing" + : ["checkpoint-binding-error", "file-checkpoint-binding-error"].includes(scenario) ? "native_runner_checkpoint_binding_mismatch" + : scenario === "active-owner" ? "prior_owner_active" + : scenario === "foreign-prior-scope" ? "prior_owner_scope_mismatch" + : scenario === "unsuspended-runner" ? "prior_owner_terminal_state_indeterminate" : "durable_identity_unreadable"; + await expect(resume).rejects.toThrow(reason); + if (["missing-checkpoint", "checkpoint-binding-error", "file-checkpoint-binding-error", "foreign-task"].includes(scenario)) + expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(markerBefore); + } + if (["foreign-marker-session", "missing-marker", "malformed-marker", "unknown-marker-schema"].includes(scenario)) expect(restore).not.toHaveBeenCalled(); + else expect(restore).toHaveBeenCalledOnce(); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + expect(existsSync(join(isolatedStateDirectory, "quarantine"))).toBe( + ["foreign-marker-session", "missing-marker", "malformed-marker", "unknown-marker-schema", "foreign-prior-scope", "unsuspended-runner"].includes(scenario)); + } finally { + state.toolAuthorityDefinitions.mockReset().mockResolvedValue([]); + restore.mockRestore(); + await controller.stop(); + } + }); + it.each([ "quarantined", "large control plane", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 5deac9f947c..e19a19689b0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4997,6 +4997,7 @@ async function migrateRunnerdStateRootForExecution(input: { }): Promise { const scoped = scopedRunnerdStateRoot(input.execution); await restoreWarmRunnerdController(input); + await restorePerTurnRunnerdController(input); // A crash can leave a source archive intent before/after its rename. Until // that exact maintenance is settled, absence of a canonical root is not // permission to bootstrap a replacement provider session. @@ -5684,6 +5685,24 @@ function idleWarmNativeSessionOwnerRunId(input: { : undefined; } +async function restorePerTurnRunnerdController(input: { + db: Db; execution: NativeExecutionInput; + runnerExecutionTarget?: AdapterExecutionTarget | null; + restartRecovery?: NativeRestartRecoveryClaim; +}): Promise { + if (input.restartRecovery || input.runnerExecutionTarget?.kind !== "remote" || + input.execution.session.lifecyclePolicy.mode !== "per_turn") return; + const root = scopedRunnerdStateRoot(input.execution); + const identity = readRunnerdDurableIdentity(root); + const reference = identity ?? readRunnerdDurableIdentity(root, true); + if (!durableIdentityMatchesSession(reference, input.execution)) return; + if (identity?.runId === input.execution.binding.runId) return; + const restored = await restoreNativeRunnerCheckpoint(input.db, + { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, runId: reference.runId }, + root, nativeSessionKey(input.execution)); + if (!restored && !identity) throw new Error("native_runner_checkpoint_missing"); +} + /** Materialize opaque controller state before ordinary warm admission reads it. * A retained owner or server checkpoint identifies the last completed run; * the original external marker can precede several in-process warm turns. */