diff --git a/doc/recoverable-disk-cleanup.md b/doc/recoverable-disk-cleanup.md index 81c3e6cb532..e6e450bd37d 100644 --- a/doc/recoverable-disk-cleanup.md +++ b/doc/recoverable-disk-cleanup.md @@ -51,8 +51,13 @@ this worker never scans or removes them. After each remote turn, the server copies the sandbox's harness state (runner state plus the provider's session home, such as `codex-home`) to `/runtime/paperclip-runner/durable-sessions//failover-backups/current`. -A replacement sandbox restores the session from this copy. Each session keeps one -copy, and a single copy can reach several gigabytes. +A replacement sandbox restores the session from this copy, and an ephemeral sandbox +restores it at the start of every turn. Each session keeps one copy, and a single +copy can reach several gigabytes. Codex copies omit credentials, scratch +directories, and data Codex downloads again on startup: remote plugin bundles under +`plugins/cache/`, the remote plugin catalog cache, and plugin +install staging. After a restore, remote plugins can be unavailable to a turn until +Codex finishes downloading them. An hourly background sweep expires the copy once the session can only resume by reopening closed work. It checks at most 200 sessions and expires at most 50 per diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c1286212309..d2b6b7eeb21 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1787,12 +1787,26 @@ describe("native harness persistence profiles", () => { }, ); - it("excludes disposable Codex scratch trees and launch-time credentials", () => { + it("excludes disposable Codex scratch trees, launch-time credentials and re-downloadable remote plugins", () => { const codex = profile({ kind: "codex" }, "codex_app_server"); expect( codex.directories.find((directory) => directory.name === "codex-home"), ).toMatchObject({ - excludeEntries: ["tmp", ".tmp", "auth.json", "config.toml"], + excludeEntries: [ + "tmp", + ".tmp", + "auth.json", + "config.toml", + ".remote-plugin-install-staging", + "cache/remote_plugin_catalog", + "plugins/.remote-plugin-install-staging", + "plugins/cache/openai-curated-remote", + "plugins/cache/created-by-me-remote", + "plugins/cache/workspace-directory", + "plugins/cache/workspace-shared-with-me", + "plugins/cache/workspace-shared-with-me-private", + "plugins/cache/workspace-shared-with-me-unlisted", + ], }); }); @@ -1807,6 +1821,15 @@ describe("native harness persistence profiles", () => { `acpx/${sessionDirectory}/codex-home/.tmp`, `acpx/${sessionDirectory}/codex-home/auth.json`, `acpx/${sessionDirectory}/codex-home/config.toml`, + `acpx/${sessionDirectory}/codex-home/.remote-plugin-install-staging`, + `acpx/${sessionDirectory}/codex-home/cache/remote_plugin_catalog`, + `acpx/${sessionDirectory}/codex-home/plugins/.remote-plugin-install-staging`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/openai-curated-remote`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/created-by-me-remote`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/workspace-directory`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/workspace-shared-with-me`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/workspace-shared-with-me-private`, + `acpx/${sessionDirectory}/codex-home/plugins/cache/workspace-shared-with-me-unlisted`, ], }); expect( @@ -2530,6 +2553,51 @@ describe("remote provider checkpoint snapshots", () => { expect(await readFile(join(home, "logs", "unified.jsonl"), "utf8")).toBe("AUTH_SENTINEL"); } finally { await rm(root, { recursive: true, force: true }); } }); + it("keeps Codex session state while omitting re-downloadable remote plugins from the checkpoint", async () => { + const root = await mkdtemp(join(tmpdir(), "codex-plugin-checkpoint-")); + try { + const source = join(root, "codex-home"); + const target = join(root, "backup"); + const files: Record = { + "sessions/2026/10/20/rollout.jsonl": "resume-state", + "state_5.sqlite": "state", + "cache/codex_apps_tools/tools.json": "apps", + "cache/remote_plugin_catalog/catalog.json": "catalog", + "plugins/data/sites/data.json": "plugin-data", + "plugins/cache/openai-bundled/browser/1.0.0/plugin.json": "bundled", + "plugins/cache/openai-curated-remote/sites/1.0.2/plugin.json": "remote", + "plugins/cache/workspace-shared-with-me/notes/0.1.0/plugin.json": "remote", + "plugins/.remote-plugin-install-staging/partial/plugin.json": "staging", + ".remote-plugin-install-staging/partial/plugin.json": "staging", + }; + for (const [path, contents] of Object.entries(files)) { + await mkdir(join(source, path, ".."), { recursive: true }); + await writeFile(join(source, path), contents); + } + const codexProfile = resolveNativeHarnessPersistenceProfile({ + provider: { kind: "codex" }, + session: { normalizedSessionId: "session", driverKind: "codex_app_server" }, + } as unknown as NativeExecutionInputV1); + const execute = async ({ command, args }: { command: string; args: string[] }) => ({ + exitCode: 0, timedOut: false, stdout: execFileSync(command, args, { encoding: "utf8" }), stderr: "", + }); + const syncOut = async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string }> }>) => { + for (const operation of operations) for (const file of operation.files) + await cp(file.sourcePath, file.targetPath, { recursive: true }); + }; + await syncRemoteRunnerDirectoryOut({ runner: { execute, syncOut } as never, + sourcePath: source, targetPath: target, mode: 0o700, + excludeEntries: codexProfile.directories.find(directory => directory.name === "codex-home")!.excludeEntries }); + for (const kept of ["sessions/2026/10/20/rollout.jsonl", "state_5.sqlite", "cache/codex_apps_tools/tools.json", + "plugins/data/sites/data.json", "plugins/cache/openai-bundled/browser/1.0.0/plugin.json"]) + expect(await readFile(join(target, kept), "utf8")).toBe(files[kept]); + for (const omitted of ["cache/remote_plugin_catalog", "plugins/cache/openai-curated-remote", + "plugins/cache/workspace-shared-with-me", "plugins/.remote-plugin-install-staging", ".remote-plugin-install-staging"]) + await expect(access(join(target, omitted))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(join(source, "plugins/cache/openai-curated-remote/sites/1.0.2/plugin.json"), "utf8")).toBe("remote"); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("excludes Codex scratch and credential state without mutating the live provider home", async () => { const execute = vi .fn() diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b28eec17206..fbe8d832d1d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -336,6 +336,24 @@ const CODEX_HOME_NON_PERSISTENT_ENTRIES = [ "auth.json", "config.toml", ] as const; +const CODEX_REMOTE_PLUGIN_MARKETPLACES = [ + "openai-curated-remote", + "created-by-me-remote", + "workspace-directory", + "workspace-shared-with-me", + "workspace-shared-with-me-private", + "workspace-shared-with-me-unlisted", +] as const; +const CODEX_HOME_REDOWNLOADABLE_ENTRIES = [ + ".remote-plugin-install-staging", + "cache/remote_plugin_catalog", + "plugins/.remote-plugin-install-staging", + ...CODEX_REMOTE_PLUGIN_MARKETPLACES.map((marketplace) => `plugins/cache/${marketplace}`), +] as const; +const CODEX_HOME_CHECKPOINT_EXCLUDED_ENTRIES = [ + ...CODEX_HOME_NON_PERSISTENT_ENTRIES, + ...CODEX_HOME_REDOWNLOADABLE_ENTRIES, +] as const; const RUNNERD_CONTROL_PLANE_STATE_SCHEMA = "paperclip.runner.durable.control-plane-state.v1"; const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1"; @@ -5518,7 +5536,7 @@ export function resolveNativeHarnessPersistenceProfile( // credential source and config.toml can contain the native MCP // bearer token. Re-materialize both for a replacement sandbox // instead of putting credentials into the disaster-recovery copy. - excludeEntries: CODEX_HOME_NON_PERSISTENT_ENTRIES, + excludeEntries: CODEX_HOME_CHECKPOINT_EXCLUDED_ENTRIES, }) : execution.provider.kind === "opencode" ? { @@ -5541,7 +5559,7 @@ export function resolveNativeHarnessPersistenceProfile( ? ["auth.json", "auth-refresh.json", "auth-refresh.json.tmp", "config.toml", "logs"].map((entry) => `acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/grok-home/${entry}`) : execution.provider.agent === "codex" - ? CODEX_HOME_NON_PERSISTENT_ENTRIES.map( + ? CODEX_HOME_CHECKPOINT_EXCLUDED_ENTRIES.map( (entry) => `acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/codex-home/${entry}`, )