From 4577d1002994690ccfba6f58e2142364f3bc5715 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:49:48 -0500 Subject: [PATCH] fix: prepare everyday artifact and Codex sandbox prerequisites in CI (#13516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The runner E2E workflow executes paid everyday workflow stories on disposable CI hosts > - The everyday artifact oracle requires a pinned Python image and fails closed when it is absent > - Fresh CI hosts did not prepare this image before the paid cell, so project stories failed during preflight > - This pull request prepares and verifies the pinned image before the affected everyday cells > - The benefit is reliable artifact isolation checks on fresh trusted CI hosts ## Linked Issues or Issue Description **What happened?** Fresh trusted CI runners did not have the pinned Python artifact oracle image. **Expected behavior:** The workflow prepares and verifies the pinned image before an everyday project story starts. **Steps to reproduce:** Run an everyday project story on a fresh CI host without the image cached. The `everyday-artifact.py --preflight` check fails before task creation. **Paperclip version or commit:** `master` at `bd51f157e`. **Deployment mode:** Other: GitHub Actions trusted paid workflow. ## What Changed - Add a matrix-gated CI step for everyday project and recovery cells. - Check Docker, pull the fixed digest with bounded timeouts, and verify the exact repo digest. - Apply the existing Codex sandbox preparation to both native Codex profiles, including the mini profile. - Add workflow security assertions for ordering, condition, digest, timeouts, and secret isolation. - Document that CI prepares the pinned oracle image. - Check provisioning eligibility against every catalog cell, and scope the Daytona registry inspection assertion to the Daytona image job. ## Verification - `pnpm test:e2e:runner:unit` — 24 files and 222 tests passed. - `pnpm test:e2e:runner:typecheck` — passed. - `pnpm exec vitest run --config tests/runner-e2e/vitest.config.ts workflow-security.test.ts` — 10 tests passed. - Python artifact oracle calibration — 12/12 passed. - `git diff --check` — passed. ## Risks Low risk. The image step runs only for everyday cells that execute the artifact preflight. The Codex setup now covers both native Codex profiles. It uses a fixed public image digest and has no provider credentials. ## Model Used OpenAI gpt-5.6-luna (implementation subagent) and gpt-6-astra (review fixes and orchestration), using code execution and repository tools. Context window size is not exposed by this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip Co-authored-by: Paperclip --- .github/workflows/runner-full-stack-e2e.yml | 11 +++++- tests/runner-e2e/EVERYDAY-WORKFLOWS.md | 6 ++- tests/runner-e2e/codex-ci-sandbox.test.ts | 41 +++++++++++++++++++++ tests/runner-e2e/codex-ci-sandbox.ts | 7 ++++ tests/runner-e2e/daytona-image.test.ts | 6 ++- tests/runner-e2e/launch.ts | 4 +- tests/runner-e2e/workflow-security.test.ts | 35 ++++++++++++++++++ 7 files changed, 104 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 2e8859971d5..d7d64335dc8 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -971,7 +971,7 @@ jobs: # This definition executes only from the authorized default-branch workflow. # Provision host policy before credentials reach target-controlled tests. - name: Provision Codex sandbox on the disposable trusted runner - if: matrix.environmentId == 'local' && matrix.profileId == 'runner-codex' + if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini') run: | node --input-type=module <<'NODE' import { execFileSync } from "node:child_process"; @@ -1002,6 +1002,15 @@ jobs: execFileSync("sudo", ["-n", "apparmor_parser", "-r", profilePath], {timeout:15000, stdio:"pipe"}); NODE + - name: Prepare pinned Python artifact oracle image + if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') + run: | + set -euo pipefail + oracle_image='python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285' + timeout 30s docker version --format '{{.Server.Version}}' + timeout 120s docker pull "$oracle_image" + timeout 30s docker image inspect "$oracle_image" --format '{{.Id}}' + - name: Run paid cell env: OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index 98abb3d5b2f..bd66843ac48 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -51,8 +51,10 @@ pnpm test:e2e:runner -- --suite everyday-workflows --environment daytona --max-p ``` Before project stories or the Python calibration tests, start Docker on the -harness host and fetch the pinned oracle image. This is required for local and -Daytona stories; artifact checks run on the harness host. +harness host and fetch the pinned oracle image. CI prepares and verifies this +same pinned image before the paid project-story cells; artifact checks run on +the harness host. The workflow verifies the exact repository digest after the +pull. This is required for local and Daytona stories. ```sh docker pull python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.ts index bf663855f05..20bd4f63f27 100644 --- a/tests/runner-e2e/codex-ci-sandbox.test.ts +++ b/tests/runner-e2e/codex-ci-sandbox.test.ts @@ -2,9 +2,49 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; + const root = path.resolve(import.meta.dirname, "../.."); describe("Codex CI sandbox trust boundary", () => { + it.each([ + ["everyday-workflows.runner-codex.local.build-revise", true], + ["everyday-workflows.runner-codex-mini.local.build-revise", true], + ["everyday-workflows.runner-acpx-claude.local.build-revise", false], + ["everyday-workflows.runner-codex.daytona.build-revise", false], + ["core-compatibility.legacy-codex.local.message-marker", false], + ] as const)("qualifies the native local Codex sandbox for %s: %s", (id, expected) => { + const execution = runnerMatrix.find((cell) => cell.id === id); + expect(execution, id).toBeDefined(); + expect(requiresCodexCiSandbox(execution!)).toBe(expected); + }); + + it("keeps trusted CI provisioning in sync with every catalog cell", async () => { + const workflow = await readFile( + path.join(root, ".github/workflows/runner-full-stack-e2e.yml"), + "utf8", + ); + const condition = workflow.match( + /- name: Provision Codex sandbox on the disposable trusted runner\n\s+if: ([^\n]+)/, + )?.[1]; + expect(condition).toMatch( + /^matrix\.environmentId == 'local' && \(matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/, + ); + const provisionedProfiles = new Set( + [...condition!.matchAll(/matrix\.profileId == '([^']+)'/g)].map((match) => match[1]), + ); + expect([...provisionedProfiles].sort()).toEqual( + [...new Set(runnerMatrix.filter(requiresCodexCiSandbox).map((cell) => cell.profile.id))].sort(), + ); + for (const cell of runnerMatrix) { + expect( + cell.environment.id === "local" && provisionedProfiles.has(cell.profile.id), + cell.id, + ).toBe(requiresCodexCiSandbox(cell)); + } + }); + it("keeps privileged policy changes out of target-controlled tests", async () => { const source = await readFile( path.join(root, "tests/runner-e2e/codex-ci-sandbox.ts"), @@ -40,6 +80,7 @@ describe("Codex CI sandbox trust boundary", () => { ); expect(paid).toBeGreaterThan(setup); const step = workflow.slice(setup, paid); + expect(step).toContain("matrix.profileId == 'runner-codex-mini'"); expect(step).toContain('binary.startsWith(root + "/node_modules/.pnpm/")'); expect(step).toContain("binary.endsWith(suffix)"); expect(step).toContain('"-n", "apparmor_parser", "-r", profilePath'); diff --git a/tests/runner-e2e/codex-ci-sandbox.ts b/tests/runner-e2e/codex-ci-sandbox.ts index 921f647b533..4e1e245bc60 100644 --- a/tests/runner-e2e/codex-ci-sandbox.ts +++ b/tests/runner-e2e/codex-ci-sandbox.ts @@ -2,6 +2,13 @@ import { execFileSync } from "node:child_process"; import { mkdir, realpath } from "node:fs/promises"; import { createRequire } from "node:module"; import path from "node:path"; +import type { MatrixExecution } from "./types.js"; + +export function requiresCodexCiSandbox(execution: MatrixExecution): boolean { + return execution.environment.id === "local" + && execution.profile.generation === "native" + && execution.profile.provider === "codex"; +} /** Probe the existing Linux sandbox before spending provider credentials. * Host policy is provisioned by the trusted workflow, never by target test code. diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index 078898a66ce..1588319845e 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -111,7 +111,11 @@ describe("runner E2E Daytona image contract", () => { expect(workflow).toContain(`docker buildx imagetools inspect "$immutable"`); expect(workflow).toContain(`--format '{{json .Image}}'`); expect(workflow).not.toContain(`docker --config "$anonymous_config" pull`); - expect(workflow).not.toContain("docker image inspect"); + const daytonaImageJob = workflow.match(/^ daytona_image:\n[\s\S]*?(?=^ \w+:)/m)?.[0]; + expect(daytonaImageJob).toBeDefined(); + // Remote Daytona manifests must use registry inspection. Local oracle + // images in the test job can still use the Docker daemon. + expect(daytonaImageJob).not.toContain("docker image inspect"); expect(workflow).not.toContain("docker buildx prune --all --force"); expect(workflow).not.toContain("docker system prune --all --force"); expect(workflow).toContain('.architecture == "amd64"'); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 40b177fd3d0..108aac9f6d6 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,5 +1,5 @@ import { randomBytes } from "node:crypto"; -import { prepareCodexCiSandbox } from "./codex-ci-sandbox.js"; +import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -647,7 +647,7 @@ async function runAttempt(input: { temporaryRoot, process.env.PATH, ); - if (execution.environment.id === "local" && execution.profile.id === "runner-codex") { + if (requiresCodexCiSandbox(execution)) { await prepareCodexCiSandbox(repositoryRoot, temporaryRoot); } const agentJwtSecret = secret(48); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index bc20ac733fa..cfaec2ee747 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -11,6 +11,8 @@ const buildRunnerNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u; const buildRemoteProviderPackNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u; +const everydayOracleImage = + "python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285"; describe("public repository paid workflow security", () => { it("uses the reviewed master branch for the first-party trusted PR workflow", async () => { @@ -280,13 +282,46 @@ describe("public repository paid workflow security", () => { const hostedChromiumInstall = paidJob.indexOf( "- name: Install Chromium headless shell on GitHub-hosted fallback", ); + const everydayOraclePreparation = paidJob.indexOf( + "- name: Prepare pinned Python artifact oracle image", + ); const paidExecution = paidJob.indexOf("- name: Run paid cell"); expect(paidInstall).toBeGreaterThan(0); expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall); expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation); expect(hostedChromiumInstall).toBeGreaterThan(awsFfmpegInstall); + expect(everydayOraclePreparation).toBeGreaterThan(hostedChromiumInstall); expect(paidExecution).toBeGreaterThan(awsFfmpegInstall); expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation); + expect(paidExecution).toBeGreaterThan(everydayOraclePreparation); + const everydayOracleStep = paidJob.slice( + everydayOraclePreparation, + paidExecution, + ); + expect(everydayOracleStep).toContain( + "if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", + ); + expect(everydayOracleStep).toContain( + `oracle_image='${everydayOracleImage}'`, + ); + expect(everydayOracleStep).toContain( + "timeout 30s docker version --format '{{.Server.Version}}'", + ); + expect(everydayOracleStep).toContain( + 'timeout 120s docker pull "$oracle_image"', + ); + expect(everydayOracleStep).toContain( + "timeout 30s docker image inspect \"$oracle_image\" --format '{{.Id}}'", + ); + const artifactSource = await readFile( + path.join(repositoryRoot, "tests/runner-e2e/everyday-artifact.py"), + "utf8", + ); + const artifactImage = artifactSource.match( + /^SANDBOX_IMAGE = '([^']+)'$/mu, + )?.[1]; + expect(artifactImage).toBe(everydayOracleImage); + expect(everydayOracleStep).not.toMatch(/secrets\./u); const awsFfmpegStep = paidJob.slice( awsFfmpegInstall, hostedChromiumInstall,