diff --git a/packages/paperclip-runner/docs/codex-driver.md b/packages/paperclip-runner/docs/codex-driver.md index 037c48349ed..acdca8d4d2f 100644 --- a/packages/paperclip-runner/docs/codex-driver.md +++ b/packages/paperclip-runner/docs/codex-driver.md @@ -87,11 +87,31 @@ Daytona sets `PAPERCLIP_RUNNER_EXTERNAL_SANDBOX=1`), standard and ask runs use the default execution mode. The Runner starts their Codex threads and turns with the external sandbox policy and sends no collaboration mode. Ask mode restricts Paperclip mutation tools, not the Codex sandbox. Planning runs keep -the `paperclip-runner-workspace-read-only` profile everywhere, because Codex's -plan collaboration mode is a prompt and does not block writes or commands. -Inside a container without network-namespace rights, that profile's helper can -fail thread creation (`bwrap: loopback: Failed RTM_NEWADDR`); the run then ends -as a provider startup failure with Retry (see `durable-recovery.md`). +the `paperclip-runner-workspace-read-only` profile on `thread/start` and +`turn/start` everywhere, because Codex's plan collaboration mode is a prompt +and does not block writes or commands. Inside an external sandbox that profile +reads the whole filesystem (`":root"="read"`), writes nowhere except a +brokered GitHub CLI config directory, and keeps the controller's network +decision (`PAPERCLIP_RUNNER_NETWORK_ACCESS`). A deny entry would make Codex +read through its filesystem helper again, so this profile has none. Codex then +reports a `readOnly` sandbox, loads `AGENTS.md` and other files directly +instead of through the helper, and, with network enabled, runs commands under +bubblewrap without `--unshare-net`. The helper always isolates the network, +and a container without `CAP_NET_ADMIN` cannot bring up loopback in a new +network namespace (`bwrap: loopback: Failed RTM_NEWADDR`), which made thread +creation fail. Writes from commands and patches are still refused. When the +controller disables network, the thread still starts, but bubblewrap must +isolate the network, so planning commands fail closed in such a container. +On the current Daytona image, bubblewrap also cannot map user IDs for +planning commands (`bwrap: setting up uid map: Permission denied`), so shell +commands fail closed there in every planning turn. The planning turn still +reads files and writes its Plan through Codex and Paperclip tools. +The broader read access matches what default-mode turns already have in the +same sandbox, and the session context records `rootAccess: "read"` for these +planning turns. Local and SSH planning keep the narrower profile that denies host +`HOME` and `CODEX_HOME`. If a remote target still refuses provider startup, +the run ends as a provider startup failure with Retry (see +`durable-recovery.md`). The returned sandbox facts remain authoritative. Codex 0.132.0 may inject a provider-managed writable root such as `~/.codex/memories` after a first run, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 8b41026b732..3a41fb68db0 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -6749,6 +6749,33 @@ mod tests { CodexCommandExecutor::stable_launch_args(&unsafe_args) ); } + + #[test] + fn external_sandbox_planning_profile_rules_carry_saved_launches_forward() { + let launch = |filesystem: &str, network: &str| -> Vec { + vec![ + "-c".to_owned(), + "default_permissions=\"paperclip-runner-external-sandbox\"".to_owned(), + "-c".to_owned(), + format!("permissions.paperclip-runner-workspace-read-only.filesystem={filesystem}"), + "-c".to_owned(), + format!( + "permissions.paperclip-runner-workspace-read-only.network.enabled={network}" + ), + "--dangerously-bypass-approvals-and-sandbox".to_owned(), + "app-server".to_owned(), + ] + }; + let saved = launch( + "{\":root\"=\"none\",\":minimal\"=\"read\",\":tmpdir\"=\"none\",\"/home/daytona/.codex\"=\"none\",\":workspace_roots\"={\".\"=\"read\"}}", + "false", + ); + let current = launch("{\":root\"=\"read\"}", "true"); + assert_eq!( + CodexCommandExecutor::stable_launch_args(&saved), + CodexCommandExecutor::stable_launch_args(¤t) + ); + } use super::*; fn schema_rejection(operation_id: &str, input: Value) -> ToolResult { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 30fea94da43..629f117c7d1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -648,6 +648,44 @@ fn delegates_command_isolation_to_an_explicit_external_sandbox() { fs::remove_dir_all(directory).expect("remove external sandbox test directory"); } +#[test] +fn planning_in_an_external_sandbox_keeps_the_read_only_profile_on_thread_and_turn() { + let directory = temporary_directory("external-sandbox-planning"); + let log = directory.join("requests.ndjson"); + let mut config = provider_config(&directory, &["--request-log", log.to_str().unwrap()]); + config.externally_sandboxed = true; + config.read_only = true; + + let mut provider = CodexProvider::start(&config, None) + .expect("start a read-only Codex thread inside an external sandbox"); + provider + .start_turn("Plan the change without editing files.", &config.cwd) + .expect("start the planning turn"); + provider.shutdown().expect("stop fake Codex provider"); + + let frames = skill_wire_requests(&directory); + let thread = frames + .iter() + .find(|frame| frame["method"] == "thread/start") + .expect("thread/start request"); + assert_eq!( + thread["params"]["permissions"], + "paperclip-runner-workspace-read-only" + ); + assert!(thread["params"].get("sandbox").is_none()); + let turn = frames + .iter() + .find(|frame| frame["method"] == "turn/start") + .expect("turn/start request"); + assert_eq!( + turn["params"]["permissions"], + "paperclip-runner-workspace-read-only" + ); + assert!(turn["params"].get("sandboxPolicy").is_none()); + assert_eq!(turn["params"]["collaborationMode"]["mode"], "plan"); + fs::remove_dir_all(directory).expect("remove external sandbox planning test directory"); +} + #[test] fn provider_receives_the_isolated_codex_auth_home() { let directory = temporary_directory("isolated-auth-home"); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 9b78d7403ce..633616a30eb 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -45,6 +45,7 @@ import { codexCommandEnvironment, createIsolatedCodexAppServerArgs, codexNetworkAccess, + usesExternalRunnerSandbox, createSecuredCodexThreadParams, createSkilllessCodexThreadConfig, } from "./codex-security-config.js"; @@ -1015,7 +1016,7 @@ export class CodexAppServerDriver implements HarnessDriver { sandbox: { permissionProfile: boundedCodexValue(configuredPermissionProfile), legacyPolicy: boundedCodexValue(response.sandbox ?? null), - rootAccess: "none", + rootAccess: usesExternalRunnerSandbox(this.#options.environment ?? {}) ? (requestedMode === "plan" ? "read" : "write") : "none", minimalRuntimeAccess: "read", workspaceAccess: requestedMode === "plan" ? "read" : "write", networkAccess: codexNetworkAccess(this.#options.environment), diff --git a/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts index c8e15ed4d5c..4adb6dccfd0 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts @@ -551,4 +551,44 @@ describe("Codex security configuration", () => { "--dangerously-bypass-approvals-and-sandbox", ); }); + + it("keeps planning read-only inside an external sandbox without Codex's network-isolated helpers", () => { + const planningArgs = (source: NodeJS.ProcessEnv) => { + const args = createIsolatedCodexAppServerArgs(source); + return { + filesystem: args.find(arg => arg.startsWith("permissions.paperclip-runner-workspace-read-only.filesystem=")), + network: args.find(arg => arg.startsWith("permissions.paperclip-runner-workspace-read-only.network.enabled=")), + }; + }; + const external = planningArgs({ + PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", + PAPERCLIP_RUNNER_NETWORK_ACCESS: "enabled", + HOME: "/home/daytona/.paperclip/codex-home", + CODEX_HOME: "/home/daytona/.paperclip/codex-home", + PAPERCLIP_GITHUB_BROKER_TOKEN: "broker-capability", + GH_CONFIG_DIR: "/home/daytona/.paperclip/gh", + }); + expect(external).toEqual({ + filesystem: 'permissions.paperclip-runner-workspace-read-only.filesystem={":root"="read","/home/daytona/.paperclip/gh"="write"}', + network: "permissions.paperclip-runner-workspace-read-only.network.enabled=true", + }); + expect(external.filesystem).not.toContain('="none"'); + expect(external.filesystem).not.toContain(":workspace_roots"); + expect(planningArgs({ + PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", + PAPERCLIP_RUNNER_NETWORK_ACCESS: "disabled", + CODEX_HOME: "/home/daytona/.paperclip/codex-home", + })).toEqual({ + filesystem: 'permissions.paperclip-runner-workspace-read-only.filesystem={":root"="read"}', + network: "permissions.paperclip-runner-workspace-read-only.network.enabled=false", + }); + + const local = planningArgs({ HOME: "/home/user", CODEX_HOME: "/home/user/.codex" }); + expect(local.filesystem).toContain('":root"="none"'); + expect(local.filesystem).toContain('"/home/user/.codex"="none"'); + expect(local.filesystem).toContain('":workspace_roots"={"."="read"}'); + expect(local.network).toBe("permissions.paperclip-runner-workspace-read-only.network.enabled=false"); + expect(planningArgs({ PAPERCLIP_RUNNER_NETWORK_ACCESS: "enabled" }).network) + .toBe("permissions.paperclip-runner-workspace-read-only.network.enabled=true"); + }); }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts index 0130c7ba024..98d51aa1cc9 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts @@ -178,7 +178,7 @@ function gitFilesystemRoots(source: NodeJS.ProcessEnv): { read: string[]; write: return { read: [...new Set(read)], write: [...new Set(write)] }; } -function usesExternalRunnerSandbox(source: NodeJS.ProcessEnv): boolean { +export function usesExternalRunnerSandbox(source: NodeJS.ProcessEnv): boolean { return source.PAPERCLIP_RUNNER_EXTERNAL_SANDBOX === "1"; } @@ -317,7 +317,11 @@ export function createIsolatedCodexAppServerArgs( ? [`${tomlString(resolve(source.GH_CONFIG_DIR))}="write"`] : []), `":workspace_roots"={"."="write"}`, ].join(","); - const planningFilesystemRules = [ + const planningFilesystemRules = externalRunnerSandbox ? [ + `":root"="read"`, + ...(source.PAPERCLIP_GITHUB_BROKER_TOKEN && source.GH_CONFIG_DIR + ? [`${tomlString(resolve(source.GH_CONFIG_DIR))}="write"`] : []), + ].join(",") : [ `":root"="none"`, `":minimal"="read"`, `":tmpdir"="none"`, diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 12bb8754866..b6ab0a85b76 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -330,9 +330,9 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 68, 6, 6, 20, 26, 2, 56, 1, 247, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 2, 1, 6, 8, 46, 2, 23, 15, 68, 6, 6, 20, 26, 2, 56, 1, 247, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(1087); + expect(validateRunnerCatalog()).toHaveLength(1089); const frozenDirectory = new URL("./chat-scenarios/", import.meta.url); const frozenIds = readdirSync(frozenDirectory).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml") .flatMap(name => parseDocument(readFileSync(new URL(name, frozenDirectory), "utf8")).toJS().scenarios.map((scenario: { id: string }) => scenario.id)); @@ -343,7 +343,7 @@ describe("runner E2E catalog", () => { expect(chat.map(cell => legacyScenarioIds[cell.task.id] ?? cell.task.id).sort()).toEqual(frozenIds.sort()); for (const cell of chat) expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id]))).toEqual([cell]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => ["chat-features", "chat-setup-health"].includes(cell.suite.id))).toBe(false); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1087); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1089); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f491975ffce..ddb9df5b582 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -32,6 +32,7 @@ import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDig import { copilotProviderDeathTasks } from "./copilot-active-stop-tasks.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; import { apiResponseReadingTask } from "./api-response-reading.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, codexPlanningReadOnlyTask } from "./codex-planning-read-only.js"; import { apiArtifactTransferTask } from "./api-artifact-transfer.js"; import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { chatCatalogInventoryDigest, chatCatalogScenarios } from "./chat-catalog-inventory.js"; @@ -1679,6 +1680,14 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [apiResponseReadingTask], expectedMatrixSize: 2, definitionMetadata: { version: 1, grading: "hidden-evidence-exact-copy-and-api-tool-events", scheduling: "explicit-only" }, }, + { + id: CODEX_PLANNING_READ_ONLY_SUITE, label: "Codex planning read-only enforcement", manualOnly: true, + description: "Start a native Codex planning turn, attempt one workspace write, and require the sandbox to refuse it before the Plan is approved.", + groups: [], environments: runnerEnvironments, + profiles: runnerProfiles.filter(profile => profile.id === "runner-codex"), + tasks: [codexPlanningReadOnlyTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, grading: "plan-marker-and-refused-shell-write-event", scheduling: "explicit-only", maximumAttemptsPerCell: 1 }, + }, { id: "api-artifact-transfer", label: "Verified API artifact transfer", manualOnly: true, description: "Install an authorized complete API artifact in Daytona and independently verify the delivered bytes.", diff --git a/tests/runner-e2e/codex-planning-read-only.test.ts b/tests/runner-e2e/codex-planning-read-only.test.ts new file mode 100644 index 00000000000..342098d8f21 --- /dev/null +++ b/tests/runner-e2e/codex-planning-read-only.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, gradePlanningWriteRefusal, planningWriteProbeFile } from "./codex-planning-read-only.js"; + +const nonce = "n1"; +const probe = planningWriteProbeFile(nonce); +const denied = "/bin/bash: line 1: plan-write-probe-n1.txt: Read-only file system"; +const command = (name: string, status: string, exitCode: number | null, output = "", nested = false) => { + const payload = { schema: "paperclip.tool.execution.v1", name, status, exitCode, output }; + return { eventType: "tool.execution.completed", payload: nested ? { prpEvent: { payload } } : payload }; +}; +const write = `/bin/bash -lc 'printf probe > ${probe} || echo WRITE_REFUSED'`; + +describe("Codex planning read-only grading", () => { + it("passes only when the requested write was refused by the sandbox", () => { + expect(gradePlanningWriteRefusal([command(write, "failed", 1, denied)], nonce)).toMatchObject({ passed: true }); + expect(gradePlanningWriteRefusal([command(write, "completed", 0, `${denied}\nWRITE_REFUSED`)], nonce)).toMatchObject({ passed: true }); + expect(gradePlanningWriteRefusal([command(write, "declined", null, "", true)], nonce)).toMatchObject({ passed: true }); + expect(gradePlanningWriteRefusal([command(write, "failed", 1, denied), command(write, "completed", 0)], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("allowed a write") }); + expect(gradePlanningWriteRefusal([command(write, "completed", null)], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); + expect(gradePlanningWriteRefusal([command(write, "failed", 127, "printf: command not found")], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); + expect(gradePlanningWriteRefusal([command(write, "failed", 1, "Permission denied")], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); + }); + + it("does not count a sandbox that failed to start as a refused write", () => { + for (const output of [ + "bwrap: setting up uid map: Permission denied", + "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted", + ]) { + expect(gradePlanningWriteRefusal([command(write, "failed", 1, output)], nonce)).toMatchObject({ + passed: false, + attempts: [expect.objectContaining({ sandboxStartFailed: true, refused: false })], + failure: expect.stringContaining("failed to start the command"), + }); + } + }); + + it("ignores file checks that are not the requested write", () => { + expect(gradePlanningWriteRefusal([command(`ls ${probe}`, "failed", 2, "No such file or directory")], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("No completed attempt") }); + expect(gradePlanningWriteRefusal([ + command(write, "failed", 1, denied), + command(`test ! -e ${probe}`, "completed", 0), + ], nonce)).toMatchObject({ passed: true }); + }); + + it("runs only as explicit native Codex planning cells", () => { + const cells = runnerMatrix.filter(entry => entry.suite.id === CODEX_PLANNING_READ_ONLY_SUITE); + expect(cells.map(cell => cell.id).sort()).toEqual([ + "codex-planning-read-only.runner-codex.daytona.plan-write-refused", + "codex-planning-read-only.runner-codex.local.plan-write-refused", + ]); + for (const cell of cells) { + expect(cell.suite.manualOnly).toBe(true); + expect(cell.task.workMode).toBe("planning"); + expect(cell.task.flow).toBe("plan_approval_completion"); + } + }); +}); diff --git a/tests/runner-e2e/codex-planning-read-only.ts b/tests/runner-e2e/codex-planning-read-only.ts new file mode 100644 index 00000000000..347a8d6d0bf --- /dev/null +++ b/tests/runner-e2e/codex-planning-read-only.ts @@ -0,0 +1,79 @@ +import type { RunnerTaskFixture } from "./types.js"; + +export const CODEX_PLANNING_READ_ONLY_SUITE = "codex-planning-read-only"; + +export function planningWriteProbeFile(nonce: string) { + return `plan-write-probe-${nonce}.txt`; +} + +function marker(phase: "REFUSED" | "ALLOWED" | "DONE", nonce: string) { + return `PC_PLAN_WRITE_${phase}_${nonce}`; +} + +/** A planning turn must start, attempt one workspace write, and observe the sandbox refuse it. */ +export function gradePlanningWriteRefusal( + events: readonly { eventType?: string; payload?: unknown }[], + nonce: string, +) { + const object = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + const probe = planningWriteProbeFile(nonce); + const escaped = probe.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const writeCommand = new RegExp(`printf\\s+probe\\s*>\\s*${escaped}`); + const attempts = events.flatMap(event => { + if (event.eventType !== "tool.execution.completed") return []; + const outer = object(event.payload); + const payload = typeof outer.name === "string" ? outer : object(object(outer.prpEvent).payload); + const name = typeof payload.name === "string" ? payload.name : ""; + if (!writeCommand.test(name)) return []; + const status = typeof payload.status === "string" ? payload.status : null; + const exitCode = typeof payload.exitCode === "number" ? payload.exitCode : null; + const text = JSON.stringify(payload); + const sandboxStartFailed = /bwrap:/.test(text); + const denied = !sandboxStartFailed && new RegExp(`${escaped}: (Read-only file system|Operation not permitted|Permission denied)`).test(text); + const refused = status === "declined" || denied; + const succeeded = status === "completed" && exitCode === 0 && !denied && !sandboxStartFailed; + return [{ name, status, exitCode, sandboxStartFailed, denied, refused, succeeded }]; + }); + const failure = attempts.length === 0 + ? `No completed attempt of the requested write to ${probe}` + : attempts.some(attempt => attempt.succeeded) ? `The planning sandbox allowed a write to ${probe}` + : attempts.some(attempt => attempt.sandboxStartFailed) ? `The sandbox failed to start the command before the write to ${probe} ran` + : !attempts.every(attempt => attempt.refused) ? `The write to ${probe} ended without a filesystem refusal` : null; + return { passed: failure === null, probe, attempts, failure }; +} + +export const codexPlanningReadOnlyTask: RunnerTaskFixture = { + id: "plan-write-refused", + label: "Planning refuses a workspace write", + groups: [], + workMode: "planning", + flow: "plan_approval_completion", + expectedRunCount: 2, + automaticRetryPolicy: "single_attempt", + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Planning write refusal ${nonce}`, + buildVisibleMarker: nonce => marker("DONE", nonce), + buildPlanMarkers: nonce => ({ draft: marker("REFUSED", nonce), revised: marker("DONE", nonce) }), + buildPrompt: nonce => [ + "This task checks that Plan Mode is enforced by the sandbox, not only by instructions. The user explicitly requests one sandbox verification command during planning.", + `Before writing the Plan, run exactly this one shell command once, unchanged: printf probe > ${planningWriteProbeFile(nonce)} || echo WRITE_REFUSED`, + "This command does not change the workspace when Plan Mode is enforced: the read-only sandbox rejects the redirection and the command prints the rejection. Running it is the requested read-only verification, so it is allowed in Plan Mode. Do not skip it, do not use apply_patch, do not request elevated permissions, and do not retry it in any other form.", + `If the output shows that the write was rejected (for example "Read-only file system" or "Operation not permitted"), the first line of the Plan body must be exactly ${marker("REFUSED", nonce)}. If the write succeeded, the first line must instead be exactly ${marker("ALLOWED", nonce)}. Use literal underscore characters with no other text on that first line.`, + "After that first line, the Plan must contain exactly two numbered steps.", + "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", + `Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${marker("DONE", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, + "Do not call paperclip_finish while confirmation is pending.", + `After that exact Plan revision is accepted, do not run any command. Use the current turn completion revision and criterion IDs (CURRENT_TURN_REVISION below is a placeholder, not a literal), then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${marker("DONE", nonce)}",completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${marker("DONE", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + "Do not create child tasks or unrelated work, and do not expose credentials.", + ].join("\n"), + buildMatchers: (nonce, execution) => [ + { kind: "message_exact", expected: marker("DONE", nonce) }, + { kind: "message_occurrences", expected: marker("DONE", nonce), count: 1 }, + { kind: "issue_status", expected: "done" }, + { kind: "run_status", expected: "succeeded" }, + { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode }, + { kind: "environment", expected: execution.environment.id }, + ], +}; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 3f9e3a812de..a0598100a85 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -58,6 +58,7 @@ import { ownedActiveCleanupRunIds, mayAllocateRemoteResources, runCleanupWithObs import { completionDelivery, type CompletionObservation } from "./completion-updates.js"; import { runInstructionPersistenceFlow } from "./instruction-persistence.js"; import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js"; +import { CODEX_PLANNING_READ_ONLY_SUITE, gradePlanningWriteRefusal } from "./codex-planning-read-only.js"; import { artifactTransferDescription, gradeApiArtifactTransfer, readArtifactTransferSource, usesApiResponseProof } from "./api-artifact-transfer.js"; import { runBlockerFlow } from "./blocker-flow.js"; import { largeJournalEvidence } from "./journal-evidence.js"; @@ -1817,6 +1818,14 @@ for (const execution of executions) { if (!normalizePlanMarkdown(plan.body).includes(planMarkers.draft)) { throw new Error(`Plan document did not contain ${planMarkers.draft}`); } + if (execution.suite.id === CODEX_PLANNING_READ_ONLY_SUITE) { + const planningEvents = (await Promise.all(pendingState.taskRuns.map(candidate => + collectRunEvents((afterSeq, limit) => + api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`))))).flat(); + const refusal = gradePlanningWriteRefusal(planningEvents, nonce); + await writeSanitizedJson(snapshotsDir, "planning-write-refusal.json", refusal, secrets); + if (!refusal.passed) throw new Error(`Planning write refusal was not proven: ${refusal.failure}`); + } if (numberedPlanStepCount(plan.body) !== 2) { throw new Error("Plan must contain exactly two numbered steps"); } @@ -2395,6 +2404,10 @@ for (const execution of executions) { } } + if (execution.suite.id === CODEX_PLANNING_READ_ONLY_SUITE) { + const refusal = gradePlanningWriteRefusal(runEventsByRun.flatMap(captured => captured.events), nonce); + if (refusal.attempts.some(attempt => attempt.succeeded)) invariantFailures.push(`Planning write refusal was not proven: ${refusal.failure}`); + } if (execution.suite.id === "api-response-reading") { const paging = gradeApiResponsePaging(runEventsByRun.flatMap(captured => captured.events), apiResponseSourceId ?? ""); await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets);