diff --git a/packages/paperclip-runner/docs/codex-driver.md b/packages/paperclip-runner/docs/codex-driver.md index a3021d88e67..f072088b16e 100644 --- a/packages/paperclip-runner/docs/codex-driver.md +++ b/packages/paperclip-runner/docs/codex-driver.md @@ -82,6 +82,23 @@ valid, but when `PAPERCLIP_WORKSPACE_CWD` is present its canonical path must be equal to or below that assigned workspace so sibling and symlink escapes fail before provider startup. +When the execution target is itself the sandbox (a remote sandbox such as +Daytona sets `PAPERCLIP_RUNNER_EXTERNAL_SANDBOX=1`), planning runs still use +the `paperclip-runner-workspace-read-only` profile on `thread/start` and +`turn/start`, because Codex's plan collaboration mode is a prompt and does not +block writes or commands. Inside that sandbox the profile reads the whole +filesystem (`":root"="read"`), writes nowhere except a brokered GitHub CLI +config directory, and enables network. Codex then reports a `readOnly` sandbox +with network access, so it loads `AGENTS.md` and other files directly instead +of through its filesystem helper, and runs commands under bubblewrap without +`--unshare-net`. The helper always isolates the network, and an unprivileged +container without `CAP_NET_ADMIN` cannot bring up loopback in a new network +namespace (`bwrap: loopback: Failed RTM_NEWADDR`), which made thread creation +fail. Writes from commands and patches are still refused. The broader read +access matches what default-mode turns already have in the same sandbox, where +Codex runs with the external sandbox policy. Local and SSH planning keep the +narrower profile that denies host `HOME` and `CODEX_HOME`. + The returned sandbox facts remain authoritative. Codex 0.132.0 may inject a provider-managed writable root such as `~/.codex/memories` after a first run, even with `features.memories=false` and an explicit Codex-home deny. That makes diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 8b41026b732..3a41fb68db0 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -6749,6 +6749,33 @@ mod tests { CodexCommandExecutor::stable_launch_args(&unsafe_args) ); } + + #[test] + fn external_sandbox_planning_profile_rules_carry_saved_launches_forward() { + let launch = |filesystem: &str, network: &str| -> Vec { + vec![ + "-c".to_owned(), + "default_permissions=\"paperclip-runner-external-sandbox\"".to_owned(), + "-c".to_owned(), + format!("permissions.paperclip-runner-workspace-read-only.filesystem={filesystem}"), + "-c".to_owned(), + format!( + "permissions.paperclip-runner-workspace-read-only.network.enabled={network}" + ), + "--dangerously-bypass-approvals-and-sandbox".to_owned(), + "app-server".to_owned(), + ] + }; + let saved = launch( + "{\":root\"=\"none\",\":minimal\"=\"read\",\":tmpdir\"=\"none\",\"/home/daytona/.codex\"=\"none\",\":workspace_roots\"={\".\"=\"read\"}}", + "false", + ); + let current = launch("{\":root\"=\"read\"}", "true"); + assert_eq!( + CodexCommandExecutor::stable_launch_args(&saved), + CodexCommandExecutor::stable_launch_args(¤t) + ); + } use super::*; fn schema_rejection(operation_id: &str, input: Value) -> ToolResult { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index c0122536536..707148346b6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -644,6 +644,44 @@ fn delegates_command_isolation_to_an_explicit_external_sandbox() { fs::remove_dir_all(directory).expect("remove external sandbox test directory"); } +#[test] +fn planning_in_an_external_sandbox_keeps_the_read_only_profile_on_thread_and_turn() { + let directory = temporary_directory("external-sandbox-planning"); + let log = directory.join("requests.ndjson"); + let mut config = provider_config(&directory, &["--request-log", log.to_str().unwrap()]); + config.externally_sandboxed = true; + config.read_only = true; + + let mut provider = CodexProvider::start(&config, None) + .expect("start a read-only Codex thread inside an external sandbox"); + provider + .start_turn("Plan the change without editing files.", &config.cwd) + .expect("start the planning turn"); + provider.shutdown().expect("stop fake Codex provider"); + + let frames = skill_wire_requests(&directory); + let thread = frames + .iter() + .find(|frame| frame["method"] == "thread/start") + .expect("thread/start request"); + assert_eq!( + thread["params"]["permissions"], + "paperclip-runner-workspace-read-only" + ); + assert!(thread["params"].get("sandbox").is_none()); + let turn = frames + .iter() + .find(|frame| frame["method"] == "turn/start") + .expect("turn/start request"); + assert_eq!( + turn["params"]["permissions"], + "paperclip-runner-workspace-read-only" + ); + assert!(turn["params"].get("sandboxPolicy").is_none()); + assert_eq!(turn["params"]["collaborationMode"]["mode"], "plan"); + fs::remove_dir_all(directory).expect("remove external sandbox planning test directory"); +} + #[test] fn provider_receives_the_isolated_codex_auth_home() { let directory = temporary_directory("isolated-auth-home"); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts index c8e15ed4d5c..1220e0d9e1b 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts @@ -551,4 +551,35 @@ describe("Codex security configuration", () => { "--dangerously-bypass-approvals-and-sandbox", ); }); + + it("keeps planning read-only inside an external sandbox without Codex's network-isolated helpers", () => { + const planningArgs = (source: NodeJS.ProcessEnv) => { + const args = createIsolatedCodexAppServerArgs(source); + return { + filesystem: args.find(arg => arg.startsWith("permissions.paperclip-runner-workspace-read-only.filesystem=")), + network: args.find(arg => arg.startsWith("permissions.paperclip-runner-workspace-read-only.network.enabled=")), + }; + }; + const external = planningArgs({ + PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", + HOME: "/home/daytona/.paperclip/codex-home", + CODEX_HOME: "/home/daytona/.paperclip/codex-home", + PAPERCLIP_GITHUB_BROKER_TOKEN: "broker-capability", + GH_CONFIG_DIR: "/home/daytona/.paperclip/gh", + }); + expect(external).toEqual({ + filesystem: 'permissions.paperclip-runner-workspace-read-only.filesystem={":root"="read","/home/daytona/.paperclip/gh"="write"}', + network: "permissions.paperclip-runner-workspace-read-only.network.enabled=true", + }); + expect(external.filesystem).not.toContain('="none"'); + expect(external.filesystem).not.toContain(":workspace_roots"); + + const local = planningArgs({ HOME: "/home/user", CODEX_HOME: "/home/user/.codex" }); + expect(local.filesystem).toContain('":root"="none"'); + expect(local.filesystem).toContain('"/home/user/.codex"="none"'); + expect(local.filesystem).toContain('":workspace_roots"={"."="read"}'); + expect(local.network).toBe("permissions.paperclip-runner-workspace-read-only.network.enabled=false"); + expect(planningArgs({ PAPERCLIP_RUNNER_NETWORK_ACCESS: "enabled" }).network) + .toBe("permissions.paperclip-runner-workspace-read-only.network.enabled=true"); + }); }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts index 0130c7ba024..20408f2ae48 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts @@ -317,7 +317,11 @@ export function createIsolatedCodexAppServerArgs( ? [`${tomlString(resolve(source.GH_CONFIG_DIR))}="write"`] : []), `":workspace_roots"={"."="write"}`, ].join(","); - const planningFilesystemRules = [ + const planningFilesystemRules = externalRunnerSandbox ? [ + `":root"="read"`, + ...(source.PAPERCLIP_GITHUB_BROKER_TOKEN && source.GH_CONFIG_DIR + ? [`${tomlString(resolve(source.GH_CONFIG_DIR))}="write"`] : []), + ].join(",") : [ `":root"="none"`, `":minimal"="read"`, `":tmpdir"="none"`, @@ -356,7 +360,7 @@ export function createIsolatedCodexAppServerArgs( "-c", `permissions.${CODEX_PLANNING_PERMISSION_PROFILE}.filesystem={${planningFilesystemRules}}`, "-c", - `permissions.${CODEX_PLANNING_PERMISSION_PROFILE}.network.enabled=${networkAccess}`, + `permissions.${CODEX_PLANNING_PERMISSION_PROFILE}.network.enabled=${networkAccess || externalRunnerSandbox}`, "-c", `shell_environment_policy.inherit=${tomlString(hasProjectedEnvironment ? "all" : "none")}`, "-c", diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 12bb8754866..b6ab0a85b76 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -330,9 +330,9 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 68, 6, 6, 20, 26, 2, 56, 1, 247, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 2, 1, 6, 8, 46, 2, 23, 15, 68, 6, 6, 20, 26, 2, 56, 1, 247, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(1087); + expect(validateRunnerCatalog()).toHaveLength(1089); const frozenDirectory = new URL("./chat-scenarios/", import.meta.url); const frozenIds = readdirSync(frozenDirectory).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml") .flatMap(name => parseDocument(readFileSync(new URL(name, frozenDirectory), "utf8")).toJS().scenarios.map((scenario: { id: string }) => scenario.id)); @@ -343,7 +343,7 @@ describe("runner E2E catalog", () => { expect(chat.map(cell => legacyScenarioIds[cell.task.id] ?? cell.task.id).sort()).toEqual(frozenIds.sort()); for (const cell of chat) expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id]))).toEqual([cell]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => ["chat-features", "chat-setup-health"].includes(cell.suite.id))).toBe(false); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1087); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1089); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f491975ffce..ddb9df5b582 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -32,6 +32,7 @@ import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDig import { copilotProviderDeathTasks } from "./copilot-active-stop-tasks.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; import { apiResponseReadingTask } from "./api-response-reading.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, codexPlanningReadOnlyTask } from "./codex-planning-read-only.js"; import { apiArtifactTransferTask } from "./api-artifact-transfer.js"; import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { chatCatalogInventoryDigest, chatCatalogScenarios } from "./chat-catalog-inventory.js"; @@ -1679,6 +1680,14 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [apiResponseReadingTask], expectedMatrixSize: 2, definitionMetadata: { version: 1, grading: "hidden-evidence-exact-copy-and-api-tool-events", scheduling: "explicit-only" }, }, + { + id: CODEX_PLANNING_READ_ONLY_SUITE, label: "Codex planning read-only enforcement", manualOnly: true, + description: "Start a native Codex planning turn, attempt one workspace write, and require the sandbox to refuse it before the Plan is approved.", + groups: [], environments: runnerEnvironments, + profiles: runnerProfiles.filter(profile => profile.id === "runner-codex"), + tasks: [codexPlanningReadOnlyTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, grading: "plan-marker-and-refused-shell-write-event", scheduling: "explicit-only", maximumAttemptsPerCell: 1 }, + }, { id: "api-artifact-transfer", label: "Verified API artifact transfer", manualOnly: true, description: "Install an authorized complete API artifact in Daytona and independently verify the delivered bytes.", diff --git a/tests/runner-e2e/codex-planning-read-only.test.ts b/tests/runner-e2e/codex-planning-read-only.test.ts new file mode 100644 index 00000000000..c0084d74491 --- /dev/null +++ b/tests/runner-e2e/codex-planning-read-only.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, gradePlanningWriteRefusal, planningWriteProbeFile } from "./codex-planning-read-only.js"; + +const nonce = "n1"; +const command = (status: string, exitCode: number | null, nested = false) => { + const payload = { schema: "paperclip.tool.execution.v1", name: `/bin/bash -lc 'printf probe > ${planningWriteProbeFile(nonce)}'`, status, exitCode }; + return { eventType: "tool.execution.completed", payload: nested ? { prpEvent: { payload } } : payload }; +}; + +describe("Codex planning read-only grading", () => { + it("passes only when the planning write attempt was refused", () => { + expect(gradePlanningWriteRefusal([command("failed", 1)], nonce)).toMatchObject({ passed: true }); + expect(gradePlanningWriteRefusal([command("declined", null, true)], nonce)).toMatchObject({ passed: true }); + expect(gradePlanningWriteRefusal([command("failed", 1), command("completed", 0)], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("allowed a write") }); + expect(gradePlanningWriteRefusal([{ eventType: "tool.execution.completed", payload: { name: "ls" } }], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("No completed shell attempt") }); + }); + + it("runs only as explicit native Codex planning cells", () => { + const cells = runnerMatrix.filter(entry => entry.suite.id === CODEX_PLANNING_READ_ONLY_SUITE); + expect(cells.map(cell => cell.id).sort()).toEqual([ + "codex-planning-read-only.runner-codex.daytona.plan-write-refused", + "codex-planning-read-only.runner-codex.local.plan-write-refused", + ]); + for (const cell of cells) { + expect(cell.suite.manualOnly).toBe(true); + expect(cell.task.workMode).toBe("planning"); + expect(cell.task.flow).toBe("plan_approval_completion"); + } + }); +}); diff --git a/tests/runner-e2e/codex-planning-read-only.ts b/tests/runner-e2e/codex-planning-read-only.ts new file mode 100644 index 00000000000..185bb67d742 --- /dev/null +++ b/tests/runner-e2e/codex-planning-read-only.ts @@ -0,0 +1,73 @@ +import type { RunnerTaskFixture } from "./types.js"; + +export const CODEX_PLANNING_READ_ONLY_SUITE = "codex-planning-read-only"; + +export function planningWriteProbeFile(nonce: string) { + return `plan-write-probe-${nonce}.txt`; +} + +function marker(phase: "REFUSED" | "ALLOWED" | "DONE", nonce: string) { + return `PC_PLAN_WRITE_${phase}_${nonce}`; +} + +/** A planning turn must start, attempt one workspace write, and observe the sandbox refuse it. */ +export function gradePlanningWriteRefusal( + events: readonly { eventType?: string; payload?: unknown }[], + nonce: string, +) { + const object = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + const probe = planningWriteProbeFile(nonce); + const attempts = events.flatMap(event => { + if (event.eventType !== "tool.execution.completed") return []; + const outer = object(event.payload); + const payload = typeof outer.name === "string" ? outer : object(object(outer.prpEvent).payload); + const name = typeof payload.name === "string" ? payload.name : ""; + if (!name.includes(probe)) return []; + return [{ + name, + status: typeof payload.status === "string" ? payload.status : null, + exitCode: typeof payload.exitCode === "number" ? payload.exitCode : null, + }]; + }); + const writeSucceeded = attempts.some(attempt => attempt.status === "completed" && attempt.exitCode === 0); + const failure = attempts.length === 0 + ? `No completed shell attempt named ${probe}` + : writeSucceeded ? `The planning sandbox allowed a write to ${probe}` : null; + return { passed: failure === null, probe, attempts, failure }; +} + +export const codexPlanningReadOnlyTask: RunnerTaskFixture = { + id: "plan-write-refused", + label: "Planning refuses a workspace write", + groups: [], + workMode: "planning", + flow: "plan_approval_completion", + expectedRunCount: 2, + automaticRetryPolicy: "single_attempt", + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Planning write refusal ${nonce}`, + buildVisibleMarker: nonce => marker("DONE", nonce), + buildPlanMarkers: nonce => ({ draft: marker("REFUSED", nonce), revised: marker("DONE", nonce) }), + buildPrompt: nonce => [ + "This task verifies that planning mode cannot change the workspace. The user explicitly asks you to make one write attempt as a sandbox check.", + `Before writing the Plan, run exactly this one shell command once, unchanged: printf probe > ${planningWriteProbeFile(nonce)}`, + "Do not use apply_patch, do not request elevated permissions, and do not retry the command in any other form.", + `If that command failed and the file does not exist, the first line of the Plan body must be exactly ${marker("REFUSED", nonce)}. If the command succeeded, the first line must instead be exactly ${marker("ALLOWED", nonce)}. Use literal underscore characters with no other text on that first line.`, + "After that first line, the Plan must contain exactly two numbered steps.", + "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", + `Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${marker("DONE", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, + "Do not call paperclip_finish while confirmation is pending.", + `After that exact Plan revision is accepted, do not run any command. Use the current turn completion revision and criterion IDs (CURRENT_TURN_REVISION below is a placeholder, not a literal), then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${marker("DONE", nonce)}",completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${marker("DONE", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + "Do not create child tasks or unrelated work, and do not expose credentials.", + ].join("\n"), + buildMatchers: (nonce, execution) => [ + { kind: "message_exact", expected: marker("DONE", nonce) }, + { kind: "message_occurrences", expected: marker("DONE", nonce), count: 1 }, + { kind: "issue_status", expected: "done" }, + { kind: "run_status", expected: "succeeded" }, + { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode }, + { kind: "environment", expected: execution.environment.id }, + ], +}; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 3f9e3a812de..c5c6daef27b 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -58,6 +58,7 @@ import { ownedActiveCleanupRunIds, mayAllocateRemoteResources, runCleanupWithObs import { completionDelivery, type CompletionObservation } from "./completion-updates.js"; import { runInstructionPersistenceFlow } from "./instruction-persistence.js"; import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, gradePlanningWriteRefusal } from "./codex-planning-read-only.js"; import { artifactTransferDescription, gradeApiArtifactTransfer, readArtifactTransferSource, usesApiResponseProof } from "./api-artifact-transfer.js"; import { runBlockerFlow } from "./blocker-flow.js"; import { largeJournalEvidence } from "./journal-evidence.js"; @@ -2395,6 +2396,11 @@ for (const execution of executions) { } } + if (execution.suite.id === CODEX_PLANNING_READ_ONLY_SUITE) { + const refusal = gradePlanningWriteRefusal(runEventsByRun.flatMap(captured => captured.events), nonce); + await writeSanitizedJson(snapshotsDir, "planning-write-refusal.json", refusal, secrets); + if (!refusal.passed) invariantFailures.push(`Planning write refusal was not proven: ${refusal.failure}`); + } if (execution.suite.id === "api-response-reading") { const paging = gradeApiResponsePaging(runEventsByRun.flatMap(captured => captured.events), apiResponseSourceId ?? ""); await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets);