diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 2e8859971d5..d7d64335dc8 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -971,7 +971,7 @@ jobs: # This definition executes only from the authorized default-branch workflow. # Provision host policy before credentials reach target-controlled tests. - name: Provision Codex sandbox on the disposable trusted runner - if: matrix.environmentId == 'local' && matrix.profileId == 'runner-codex' + if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini') run: | node --input-type=module <<'NODE' import { execFileSync } from "node:child_process"; @@ -1002,6 +1002,15 @@ jobs: execFileSync("sudo", ["-n", "apparmor_parser", "-r", profilePath], {timeout:15000, stdio:"pipe"}); NODE + - name: Prepare pinned Python artifact oracle image + if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') + run: | + set -euo pipefail + oracle_image='python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285' + timeout 30s docker version --format '{{.Server.Version}}' + timeout 120s docker pull "$oracle_image" + timeout 30s docker image inspect "$oracle_image" --format '{{.Id}}' + - name: Run paid cell env: OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index 98abb3d5b2f..bd66843ac48 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -51,8 +51,10 @@ pnpm test:e2e:runner -- --suite everyday-workflows --environment daytona --max-p ``` Before project stories or the Python calibration tests, start Docker on the -harness host and fetch the pinned oracle image. This is required for local and -Daytona stories; artifact checks run on the harness host. +harness host and fetch the pinned oracle image. CI prepares and verifies this +same pinned image before the paid project-story cells; artifact checks run on +the harness host. The workflow verifies the exact repository digest after the +pull. This is required for local and Daytona stories. ```sh docker pull python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.ts index bf663855f05..20bd4f63f27 100644 --- a/tests/runner-e2e/codex-ci-sandbox.test.ts +++ b/tests/runner-e2e/codex-ci-sandbox.test.ts @@ -2,9 +2,49 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; + const root = path.resolve(import.meta.dirname, "../.."); describe("Codex CI sandbox trust boundary", () => { + it.each([ + ["everyday-workflows.runner-codex.local.build-revise", true], + ["everyday-workflows.runner-codex-mini.local.build-revise", true], + ["everyday-workflows.runner-acpx-claude.local.build-revise", false], + ["everyday-workflows.runner-codex.daytona.build-revise", false], + ["core-compatibility.legacy-codex.local.message-marker", false], + ] as const)("qualifies the native local Codex sandbox for %s: %s", (id, expected) => { + const execution = runnerMatrix.find((cell) => cell.id === id); + expect(execution, id).toBeDefined(); + expect(requiresCodexCiSandbox(execution!)).toBe(expected); + }); + + it("keeps trusted CI provisioning in sync with every catalog cell", async () => { + const workflow = await readFile( + path.join(root, ".github/workflows/runner-full-stack-e2e.yml"), + "utf8", + ); + const condition = workflow.match( + /- name: Provision Codex sandbox on the disposable trusted runner\n\s+if: ([^\n]+)/, + )?.[1]; + expect(condition).toMatch( + /^matrix\.environmentId == 'local' && \(matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/, + ); + const provisionedProfiles = new Set( + [...condition!.matchAll(/matrix\.profileId == '([^']+)'/g)].map((match) => match[1]), + ); + expect([...provisionedProfiles].sort()).toEqual( + [...new Set(runnerMatrix.filter(requiresCodexCiSandbox).map((cell) => cell.profile.id))].sort(), + ); + for (const cell of runnerMatrix) { + expect( + cell.environment.id === "local" && provisionedProfiles.has(cell.profile.id), + cell.id, + ).toBe(requiresCodexCiSandbox(cell)); + } + }); + it("keeps privileged policy changes out of target-controlled tests", async () => { const source = await readFile( path.join(root, "tests/runner-e2e/codex-ci-sandbox.ts"), @@ -40,6 +80,7 @@ describe("Codex CI sandbox trust boundary", () => { ); expect(paid).toBeGreaterThan(setup); const step = workflow.slice(setup, paid); + expect(step).toContain("matrix.profileId == 'runner-codex-mini'"); expect(step).toContain('binary.startsWith(root + "/node_modules/.pnpm/")'); expect(step).toContain("binary.endsWith(suffix)"); expect(step).toContain('"-n", "apparmor_parser", "-r", profilePath'); diff --git a/tests/runner-e2e/codex-ci-sandbox.ts b/tests/runner-e2e/codex-ci-sandbox.ts index 921f647b533..4e1e245bc60 100644 --- a/tests/runner-e2e/codex-ci-sandbox.ts +++ b/tests/runner-e2e/codex-ci-sandbox.ts @@ -2,6 +2,13 @@ import { execFileSync } from "node:child_process"; import { mkdir, realpath } from "node:fs/promises"; import { createRequire } from "node:module"; import path from "node:path"; +import type { MatrixExecution } from "./types.js"; + +export function requiresCodexCiSandbox(execution: MatrixExecution): boolean { + return execution.environment.id === "local" + && execution.profile.generation === "native" + && execution.profile.provider === "codex"; +} /** Probe the existing Linux sandbox before spending provider credentials. * Host policy is provisioned by the trusted workflow, never by target test code. diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index 078898a66ce..1588319845e 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -111,7 +111,11 @@ describe("runner E2E Daytona image contract", () => { expect(workflow).toContain(`docker buildx imagetools inspect "$immutable"`); expect(workflow).toContain(`--format '{{json .Image}}'`); expect(workflow).not.toContain(`docker --config "$anonymous_config" pull`); - expect(workflow).not.toContain("docker image inspect"); + const daytonaImageJob = workflow.match(/^ daytona_image:\n[\s\S]*?(?=^ \w+:)/m)?.[0]; + expect(daytonaImageJob).toBeDefined(); + // Remote Daytona manifests must use registry inspection. Local oracle + // images in the test job can still use the Docker daemon. + expect(daytonaImageJob).not.toContain("docker image inspect"); expect(workflow).not.toContain("docker buildx prune --all --force"); expect(workflow).not.toContain("docker system prune --all --force"); expect(workflow).toContain('.architecture == "amd64"'); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 40b177fd3d0..108aac9f6d6 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,5 +1,5 @@ import { randomBytes } from "node:crypto"; -import { prepareCodexCiSandbox } from "./codex-ci-sandbox.js"; +import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -647,7 +647,7 @@ async function runAttempt(input: { temporaryRoot, process.env.PATH, ); - if (execution.environment.id === "local" && execution.profile.id === "runner-codex") { + if (requiresCodexCiSandbox(execution)) { await prepareCodexCiSandbox(repositoryRoot, temporaryRoot); } const agentJwtSecret = secret(48); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index bc20ac733fa..cfaec2ee747 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -11,6 +11,8 @@ const buildRunnerNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u; const buildRemoteProviderPackNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u; +const everydayOracleImage = + "python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285"; describe("public repository paid workflow security", () => { it("uses the reviewed master branch for the first-party trusted PR workflow", async () => { @@ -280,13 +282,46 @@ describe("public repository paid workflow security", () => { const hostedChromiumInstall = paidJob.indexOf( "- name: Install Chromium headless shell on GitHub-hosted fallback", ); + const everydayOraclePreparation = paidJob.indexOf( + "- name: Prepare pinned Python artifact oracle image", + ); const paidExecution = paidJob.indexOf("- name: Run paid cell"); expect(paidInstall).toBeGreaterThan(0); expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall); expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation); expect(hostedChromiumInstall).toBeGreaterThan(awsFfmpegInstall); + expect(everydayOraclePreparation).toBeGreaterThan(hostedChromiumInstall); expect(paidExecution).toBeGreaterThan(awsFfmpegInstall); expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation); + expect(paidExecution).toBeGreaterThan(everydayOraclePreparation); + const everydayOracleStep = paidJob.slice( + everydayOraclePreparation, + paidExecution, + ); + expect(everydayOracleStep).toContain( + "if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", + ); + expect(everydayOracleStep).toContain( + `oracle_image='${everydayOracleImage}'`, + ); + expect(everydayOracleStep).toContain( + "timeout 30s docker version --format '{{.Server.Version}}'", + ); + expect(everydayOracleStep).toContain( + 'timeout 120s docker pull "$oracle_image"', + ); + expect(everydayOracleStep).toContain( + "timeout 30s docker image inspect \"$oracle_image\" --format '{{.Id}}'", + ); + const artifactSource = await readFile( + path.join(repositoryRoot, "tests/runner-e2e/everyday-artifact.py"), + "utf8", + ); + const artifactImage = artifactSource.match( + /^SANDBOX_IMAGE = '([^']+)'$/mu, + )?.[1]; + expect(artifactImage).toBe(everydayOracleImage); + expect(everydayOracleStep).not.toMatch(/secrets\./u); const awsFfmpegStep = paidJob.slice( awsFfmpegInstall, hostedChromiumInstall,