mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 18:19:05 +02:00
fix(runner-e2e): provision complete worker prerequisites (#13674)
## Thinking Path
> - Paperclip manages work performed by AI agents.
> - Runner E2E tests verify that work in local and remote environments.
> - The full catalog exposed setup failures before agents could execute
tasks.
> - ACPX Codex missed sandbox provisioning, and new artifact cases
missed Docker preparation.
> - A host Claude version probe also stopped native Claude stories that
use a packaged provider.
> - This pull request repairs trusted worker setup and keeps its
selectors covered by catalog tests.
> - The resulting reruns can measure behavior instead of missing
prerequisites.
## Linked Issues or Issue Description
Follow-up to #13655. Full-catalog campaign:
https://github.com/paperclipai/paperclip/actions/runs/35417932353.
**What happened?**
ACPX Codex could not start its sandbox. New artifact stories missed
Docker preparation. Native Claude stories tried to spawn an unrelated
host CLI. The report job failed on trusted lockfile drift.
**What did you expect to happen?**
Prepare each worker's required capabilities before paid execution and
publish the retained results from trusted code.
**Steps to reproduce**
Run local ACPX Codex, everyday agent-review-handoff, or native Claude
everyday cells in the full-stack workflow at 43acbcc39.
## What Changed
- Include local ACPX Codex in exact-binary sandbox provisioning and
preflight.
- Prepare the pinned artifact oracle for agent-review-handoff. Do not
require it for skill creation.
- Remove host CLI version probes from native everyday stories.
- Retry GitHub actor lookup up to three times, with a deadline, while
retaining all authorization checks.
- Resolve reporter dependencies from the trusted checkout before its
frozen install.
- Test worker selections against the catalog and document the
default-branch requirement.
## Verification
- `pnpm test:e2e:runner:unit`: 383 tests pass.
- `pnpm test:e2e:runner:typecheck`: passes.
- PR checks: 54 passed, two intentionally skipped. Greptile: 5/5, no
inline findings.
- The paid workflow reads trusted setup from master. These setup changes
need to land before the affected Linux cells can verify them. Runtime
fixes and other affected reruns are on a separate branch.
## Risks
The setup selectors decide which workers receive sandbox policy and
Docker preparation. Catalog coverage checks their scope. Actor lookup
still fails closed. Reporter dependency resolution uses only the trusted
checkout; target branch code does not receive publication credentials.
No production prompt changes.
## Model Used
OpenAI GPT-6 via Codex, with repository inspection, code editing, and
test execution. The exact API model ID and context-window size are not
exposed in this session.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge
Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
36dbb7ed1c
commit
4cfc0e3f6c
7 files changed
+61
-13
No files matched your search
@@ -84,7 +84,15 @@ jobs:
|
||||
echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2
|
||||
exit 1
|
||||
fi
|
||||
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
|
||||
# Retry transient transport/server failures, but never proceed without
|
||||
# a successfully resolved identity and the allowlist checks below.
|
||||
for attempt in 1 2 3; do
|
||||
if triggering_actor_id="$(timeout 30s gh api "users/$TRIGGERING_ACTOR" --jq .id)"; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" = 3 ]; then exit 1; fi
|
||||
sleep "$((attempt * 2))"
|
||||
done
|
||||
if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then
|
||||
echo "GitHub actor identity contexts disagree; refusing the paid run." >&2
|
||||
exit 1
|
||||
@@ -779,7 +787,15 @@ jobs:
|
||||
set -euo pipefail
|
||||
test "$REF" = "refs/heads/$DEFAULT_BRANCH"
|
||||
jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null
|
||||
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
|
||||
# Retry transient transport/server failures, but never proceed without
|
||||
# a successfully resolved identity and the allowlist checks below.
|
||||
for attempt in 1 2 3; do
|
||||
if triggering_actor_id="$(timeout 30s gh api "users/$TRIGGERING_ACTOR" --jq .id)"; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" = 3 ]; then exit 1; fi
|
||||
sleep "$((attempt * 2))"
|
||||
done
|
||||
jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
|
||||
jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
|
||||
|
||||
@@ -976,7 +992,7 @@ jobs:
|
||||
# This definition executes only from the authorized default-branch workflow.
|
||||
# Provision host policy before credentials reach target-controlled tests.
|
||||
- name: Provision Codex sandbox on the disposable trusted runner
|
||||
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini')
|
||||
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex')
|
||||
run: |
|
||||
node --input-type=module <<'NODE'
|
||||
import { execFileSync } from "node:child_process";
|
||||
@@ -1008,7 +1024,7 @@ jobs:
|
||||
NODE
|
||||
|
||||
- name: Prepare pinned Python artifact oracle image
|
||||
if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')
|
||||
if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')
|
||||
run: |
|
||||
set -euo pipefail
|
||||
oracle_image='python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285'
|
||||
@@ -1074,6 +1090,9 @@ jobs:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
|
||||
# Resolve from this trusted checkout only. Never use the target branch's
|
||||
# lockfile or install scripts in the publication job.
|
||||
- run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Resolve workflow job attempts
|
||||
|
||||
@@ -771,3 +771,16 @@ confirmed discovery on `2.1.277`. The workflow pin and local qualifier are check
|
||||
together. This change applies to local cells; Daytona images remain separately pinned.
|
||||
Continuation question flows also wait for the submitted interaction's durable
|
||||
`answered` state before considering the next checkpoint ready.
|
||||
|
||||
### Worker prerequisites
|
||||
|
||||
The trusted default-branch workflow provisions the local Codex sandbox for both
|
||||
native Codex and ACPX Codex. It prepares the pinned Python artifact oracle only
|
||||
for everyday stories that execute a downloaded ZIP; skill creation and service
|
||||
questions do not need that oracle. Catalog coverage tests keep this list aligned
|
||||
with the test flow. Native provider runs do not require an unrelated host
|
||||
`claude` or `codex` CLI for version probing.
|
||||
|
||||
Changes to privileged worker setup must reach the default branch before a
|
||||
branch-targeted paid campaign can exercise them. The report job resolves its
|
||||
lockfile from its own trusted checkout, never from the tested branch.
|
||||
@@ -3,6 +3,7 @@ import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { requiresEverydayArtifactOracle } from "./everyday-cases.js";
|
||||
import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
|
||||
|
||||
const root = path.resolve(import.meta.dirname, "../..");
|
||||
@@ -14,6 +15,7 @@ describe("Codex CI sandbox trust boundary", () => {
|
||||
["everyday-workflows.runner-acpx-claude.local.build-revise", false],
|
||||
["everyday-workflows.runner-codex.daytona.build-revise", false],
|
||||
["core-compatibility.legacy-codex.local.message-marker", false],
|
||||
["core-compatibility.runner-acpx-codex.local.message-marker", true],
|
||||
] as const)("qualifies the native local Codex sandbox for %s: %s", (id, expected) => {
|
||||
const execution = runnerMatrix.find((cell) => cell.id === id);
|
||||
expect(execution, id).toBeDefined();
|
||||
@@ -90,3 +92,15 @@ describe("Codex CI sandbox trust boundary", () => {
|
||||
expect(step).not.toContain("sysctl -w");
|
||||
});
|
||||
});
|
||||
|
||||
it("prepares the artifact oracle for exactly the stories that execute downloads", async () => {
|
||||
const workflow = await readFile(path.join(root, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
|
||||
const condition = workflow.match(/- name: Prepare pinned Python artifact oracle image\n\s+if: ([^\n]+)/)?.[1];
|
||||
expect(condition).toContain("matrix.suiteId == 'everyday-workflows'");
|
||||
const cases = new Set([...condition!.matchAll(/matrix\.caseId == '([^']+)'/g)].map((match) => match[1]));
|
||||
for (const cell of runnerMatrix.filter((cell) => cell.suite.id === "everyday-workflows")) {
|
||||
expect(cases.has(cell.task.id), cell.id).toBe(requiresEverydayArtifactOracle(cell.task.id));
|
||||
}
|
||||
expect(requiresEverydayArtifactOracle("agent-review-handoff")).toBe(true);
|
||||
expect(requiresEverydayArtifactOracle("create-skill-studio")).toBe(false);
|
||||
});
|
||||
@@ -7,7 +7,7 @@ import type { MatrixExecution } from "./types.js";
|
||||
export function requiresCodexCiSandbox(execution: MatrixExecution): boolean {
|
||||
return execution.environment.id === "local"
|
||||
&& execution.profile.generation === "native"
|
||||
&& execution.profile.provider === "codex";
|
||||
&& (execution.profile.provider === "codex" || execution.profile.id === "runner-acpx-codex");
|
||||
}
|
||||
|
||||
/** Probe the existing Linux sandbox before spending provider credentials.
|
||||
|
||||
@@ -110,3 +110,8 @@ export const everydayTasks: readonly RunnerTaskFixture[] = definitions.map(
|
||||
buildMatchers: () => [], // The workflow records independent artifact and lifecycle checks.
|
||||
}),
|
||||
);
|
||||
|
||||
/** Only these stories execute downloaded Python ZIPs in the pinned oracle. */
|
||||
export function requiresEverydayArtifactOracle(caseId: string): boolean {
|
||||
return ["build-revise", "delegate-feedback", "agent-review-handoff", "hire-reuse", "recover-controller", "stop-redirect"].includes(caseId);
|
||||
}
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
StoryDecisionError,
|
||||
type StoryInteraction,
|
||||
} from "./everyday-decisions.js";
|
||||
import { LATE_REQUIREMENT, SLUGIFY_REVISION } from "./everyday-cases.js";
|
||||
import { LATE_REQUIREMENT, SLUGIFY_REVISION, requiresEverydayArtifactOracle } from "./everyday-cases.js";
|
||||
import {
|
||||
isActiveStoryRun,
|
||||
isStoryWorkspaceDeferral,
|
||||
@@ -502,11 +502,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
await mkdir(path.join(input.privateDir, "snapshots"), { recursive: true });
|
||||
const revision = await runCommand("git", ["rev-parse", "HEAD"]);
|
||||
if (revision.code === 0) ev.sourceRevision = revision.stdout.trim();
|
||||
const version = await runCommand(
|
||||
execution.profile.provider === "acpx" ? "claude" : "codex",
|
||||
["--version"],
|
||||
);
|
||||
if (version.code === 0) ev.providerVersion = version.stdout.trim();
|
||||
// Native providers run the packaged runtime (possibly remotely). A host
|
||||
// `claude`/`codex` binary is neither required nor its observed version.
|
||||
const harnessFiles = [
|
||||
"everyday-flow.ts",
|
||||
"everyday-cases.ts",
|
||||
@@ -536,7 +533,7 @@ export async function runEverydayFlow(input: Input) {
|
||||
.join("\n"),
|
||||
)
|
||||
.digest("hex");
|
||||
if (!caseId.startsWith("service-") && !decliningConnection) {
|
||||
if (requiresEverydayArtifactOracle(caseId)) {
|
||||
try {
|
||||
const sandbox = await runCommand(process.env.PYTHON ?? "python3", [
|
||||
path.join(import.meta.dirname, "everyday-artifact.py"), "--preflight",
|
||||
|
||||
@@ -299,7 +299,7 @@ describe("public repository paid workflow security", () => {
|
||||
paidExecution,
|
||||
);
|
||||
expect(everydayOracleStep).toContain(
|
||||
"if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')",
|
||||
"if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')",
|
||||
);
|
||||
expect(everydayOracleStep).toContain(
|
||||
`oracle_image='${everydayOracleImage}'`,
|
||||
|
||||
Reference in new issue
Block a user