From 4cfc0e3f6c7e90b7937daad6bbbf5c73942b2bd9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 19 Sep 2026 09:27:15 -0500 Subject: [PATCH] fix(runner-e2e): provision complete worker prerequisites (#13674) ## Thinking Path > - Paperclip manages work performed by AI agents. > - Runner E2E tests verify that work in local and remote environments. > - The full catalog exposed setup failures before agents could execute tasks. > - ACPX Codex missed sandbox provisioning, and new artifact cases missed Docker preparation. > - A host Claude version probe also stopped native Claude stories that use a packaged provider. > - This pull request repairs trusted worker setup and keeps its selectors covered by catalog tests. > - The resulting reruns can measure behavior instead of missing prerequisites. ## Linked Issues or Issue Description Follow-up to #13655. Full-catalog campaign: https://github.com/paperclipai/paperclip/actions/runs/35417932353. **What happened?** ACPX Codex could not start its sandbox. New artifact stories missed Docker preparation. Native Claude stories tried to spawn an unrelated host CLI. The report job failed on trusted lockfile drift. **What did you expect to happen?** Prepare each worker's required capabilities before paid execution and publish the retained results from trusted code. **Steps to reproduce** Run local ACPX Codex, everyday agent-review-handoff, or native Claude everyday cells in the full-stack workflow at 43acbcc39. ## What Changed - Include local ACPX Codex in exact-binary sandbox provisioning and preflight. - Prepare the pinned artifact oracle for agent-review-handoff. Do not require it for skill creation. - Remove host CLI version probes from native everyday stories. - Retry GitHub actor lookup up to three times, with a deadline, while retaining all authorization checks. - Resolve reporter dependencies from the trusted checkout before its frozen install. - Test worker selections against the catalog and document the default-branch requirement. ## Verification - `pnpm test:e2e:runner:unit`: 383 tests pass. - `pnpm test:e2e:runner:typecheck`: passes. - PR checks: 54 passed, two intentionally skipped. Greptile: 5/5, no inline findings. - The paid workflow reads trusted setup from master. These setup changes need to land before the affected Linux cells can verify them. Runtime fixes and other affected reruns are on a separate branch. ## Risks The setup selectors decide which workers receive sandbox policy and Docker preparation. Catalog coverage checks their scope. Actor lookup still fails closed. Reporter dependency resolution uses only the trusted checkout; target branch code does not receive publication credentials. No production prompt changes. ## Model Used OpenAI GPT-6 via Codex, with repository inspection, code editing, and test execution. The exact API model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .github/workflows/runner-full-stack-e2e.yml | 27 ++++++++++++++++++--- tests/runner-e2e/README.md | 13 ++++++++++ tests/runner-e2e/codex-ci-sandbox.test.ts | 14 +++++++++++ tests/runner-e2e/codex-ci-sandbox.ts | 2 +- tests/runner-e2e/everyday-cases.ts | 5 ++++ tests/runner-e2e/everyday-flow.ts | 11 +++------ tests/runner-e2e/workflow-security.test.ts | 2 +- 7 files changed, 61 insertions(+), 13 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 73042a12b5..7ea1660729 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -84,7 +84,15 @@ jobs: echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 exit 1 fi - triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + # Retry transient transport/server failures, but never proceed without + # a successfully resolved identity and the allowlist checks below. + for attempt in 1 2 3; do + if triggering_actor_id="$(timeout 30s gh api "users/$TRIGGERING_ACTOR" --jq .id)"; then + break + fi + if [ "$attempt" = 3 ]; then exit 1; fi + sleep "$((attempt * 2))" + done if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 exit 1 @@ -779,7 +787,15 @@ jobs: set -euo pipefail test "$REF" = "refs/heads/$DEFAULT_BRANCH" jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null - triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + # Retry transient transport/server failures, but never proceed without + # a successfully resolved identity and the allowlist checks below. + for attempt in 1 2 3; do + if triggering_actor_id="$(timeout 30s gh api "users/$TRIGGERING_ACTOR" --jq .id)"; then + break + fi + if [ "$attempt" = 3 ]; then exit 1; fi + sleep "$((attempt * 2))" + done jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null @@ -976,7 +992,7 @@ jobs: # This definition executes only from the authorized default-branch workflow. # Provision host policy before credentials reach target-controlled tests. - name: Provision Codex sandbox on the disposable trusted runner - if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini') + if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex') run: | node --input-type=module <<'NODE' import { execFileSync } from "node:child_process"; @@ -1008,7 +1024,7 @@ jobs: NODE - name: Prepare pinned Python artifact oracle image - if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') + if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') run: | set -euo pipefail oracle_image='python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285' @@ -1074,6 +1090,9 @@ jobs: node-version: 24 cache: pnpm + # Resolve from this trusted checkout only. Never use the target branch's + # lockfile or install scripts in the publication job. + - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile - run: pnpm install --frozen-lockfile - name: Resolve workflow job attempts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 5ef2169bd8..7d54c59de8 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -771,3 +771,16 @@ confirmed discovery on `2.1.277`. The workflow pin and local qualifier are check together. This change applies to local cells; Daytona images remain separately pinned. Continuation question flows also wait for the submitted interaction's durable `answered` state before considering the next checkpoint ready. + +### Worker prerequisites + +The trusted default-branch workflow provisions the local Codex sandbox for both +native Codex and ACPX Codex. It prepares the pinned Python artifact oracle only +for everyday stories that execute a downloaded ZIP; skill creation and service +questions do not need that oracle. Catalog coverage tests keep this list aligned +with the test flow. Native provider runs do not require an unrelated host +`claude` or `codex` CLI for version probing. + +Changes to privileged worker setup must reach the default branch before a +branch-targeted paid campaign can exercise them. The report job resolves its +lockfile from its own trusted checkout, never from the tested branch. diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.ts index 20bd4f63f2..b41dc24d89 100644 --- a/tests/runner-e2e/codex-ci-sandbox.test.ts +++ b/tests/runner-e2e/codex-ci-sandbox.test.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { runnerMatrix } from "./catalog.js"; +import { requiresEverydayArtifactOracle } from "./everyday-cases.js"; import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; const root = path.resolve(import.meta.dirname, "../.."); @@ -14,6 +15,7 @@ describe("Codex CI sandbox trust boundary", () => { ["everyday-workflows.runner-acpx-claude.local.build-revise", false], ["everyday-workflows.runner-codex.daytona.build-revise", false], ["core-compatibility.legacy-codex.local.message-marker", false], + ["core-compatibility.runner-acpx-codex.local.message-marker", true], ] as const)("qualifies the native local Codex sandbox for %s: %s", (id, expected) => { const execution = runnerMatrix.find((cell) => cell.id === id); expect(execution, id).toBeDefined(); @@ -90,3 +92,15 @@ describe("Codex CI sandbox trust boundary", () => { expect(step).not.toContain("sysctl -w"); }); }); + +it("prepares the artifact oracle for exactly the stories that execute downloads", async () => { + const workflow = await readFile(path.join(root, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); + const condition = workflow.match(/- name: Prepare pinned Python artifact oracle image\n\s+if: ([^\n]+)/)?.[1]; + expect(condition).toContain("matrix.suiteId == 'everyday-workflows'"); + const cases = new Set([...condition!.matchAll(/matrix\.caseId == '([^']+)'/g)].map((match) => match[1])); + for (const cell of runnerMatrix.filter((cell) => cell.suite.id === "everyday-workflows")) { + expect(cases.has(cell.task.id), cell.id).toBe(requiresEverydayArtifactOracle(cell.task.id)); + } + expect(requiresEverydayArtifactOracle("agent-review-handoff")).toBe(true); + expect(requiresEverydayArtifactOracle("create-skill-studio")).toBe(false); +}); diff --git a/tests/runner-e2e/codex-ci-sandbox.ts b/tests/runner-e2e/codex-ci-sandbox.ts index 4e1e245bc6..b0d8faf419 100644 --- a/tests/runner-e2e/codex-ci-sandbox.ts +++ b/tests/runner-e2e/codex-ci-sandbox.ts @@ -7,7 +7,7 @@ import type { MatrixExecution } from "./types.js"; export function requiresCodexCiSandbox(execution: MatrixExecution): boolean { return execution.environment.id === "local" && execution.profile.generation === "native" - && execution.profile.provider === "codex"; + && (execution.profile.provider === "codex" || execution.profile.id === "runner-acpx-codex"); } /** Probe the existing Linux sandbox before spending provider credentials. diff --git a/tests/runner-e2e/everyday-cases.ts b/tests/runner-e2e/everyday-cases.ts index 02d4955c55..11afe75035 100644 --- a/tests/runner-e2e/everyday-cases.ts +++ b/tests/runner-e2e/everyday-cases.ts @@ -110,3 +110,8 @@ export const everydayTasks: readonly RunnerTaskFixture[] = definitions.map( buildMatchers: () => [], // The workflow records independent artifact and lifecycle checks. }), ); + +/** Only these stories execute downloaded Python ZIPs in the pinned oracle. */ +export function requiresEverydayArtifactOracle(caseId: string): boolean { + return ["build-revise", "delegate-feedback", "agent-review-handoff", "hire-reuse", "recover-controller", "stop-redirect"].includes(caseId); +} diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index 590c304ae8..6885868d8d 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -18,7 +18,7 @@ import { StoryDecisionError, type StoryInteraction, } from "./everyday-decisions.js"; -import { LATE_REQUIREMENT, SLUGIFY_REVISION } from "./everyday-cases.js"; +import { LATE_REQUIREMENT, SLUGIFY_REVISION, requiresEverydayArtifactOracle } from "./everyday-cases.js"; import { isActiveStoryRun, isStoryWorkspaceDeferral, @@ -502,11 +502,8 @@ export async function runEverydayFlow(input: Input) { await mkdir(path.join(input.privateDir, "snapshots"), { recursive: true }); const revision = await runCommand("git", ["rev-parse", "HEAD"]); if (revision.code === 0) ev.sourceRevision = revision.stdout.trim(); - const version = await runCommand( - execution.profile.provider === "acpx" ? "claude" : "codex", - ["--version"], - ); - if (version.code === 0) ev.providerVersion = version.stdout.trim(); + // Native providers run the packaged runtime (possibly remotely). A host + // `claude`/`codex` binary is neither required nor its observed version. const harnessFiles = [ "everyday-flow.ts", "everyday-cases.ts", @@ -536,7 +533,7 @@ export async function runEverydayFlow(input: Input) { .join("\n"), ) .digest("hex"); - if (!caseId.startsWith("service-") && !decliningConnection) { + if (requiresEverydayArtifactOracle(caseId)) { try { const sandbox = await runCommand(process.env.PYTHON ?? "python3", [ path.join(import.meta.dirname, "everyday-artifact.py"), "--preflight", diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index cfaec2ee74..6e80147d17 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -299,7 +299,7 @@ describe("public repository paid workflow security", () => { paidExecution, ); expect(everydayOracleStep).toContain( - "if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", + "if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", ); expect(everydayOracleStep).toContain( `oracle_image='${everydayOracleImage}'`,