diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index fbb60238de3..602d9e65fee 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -50,6 +50,9 @@ jobs: pnpm --filter @paperclipai/paperclip-eval-kernel build pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals + - name: Build native Runner test binaries + run: pnpm --filter @paperclipai/paperclip-runner build:rust + - name: Run Runner fault and replay suites run: | pnpm --filter @paperclipai/paperclip-runner exec vitest run \ diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 677ff0bc812..4f9c70c647b 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -604,6 +604,18 @@ test("release verification builds native test binaries in a required Rust-cached assert.doesNotMatch(cloud, /continue-on-error|always\(\)/); }); +test("chaos verification prepares native Runner binaries before fault suites", () => { + const workflow = readWorkflow("runner-chaos-evals.yml"); + const steps = workflow.split(/\n(?= - )/); + const install = steps.find(step => /run:\s*pnpm install\b/.test(step)); + const prepare = steps.find(step => /run:\s*pnpm --filter @paperclipai\/paperclip-runner build:rust\b/.test(step)); + const faults = steps.find(step => /src\/native-session-runtime\.test\.ts/.test(step)); + assert.ok(install && prepare && faults, "the real Runner fault suite needs its native build prerequisite"); + assert.ok(steps.indexOf(install) < steps.indexOf(prepare)); + assert.ok(steps.indexOf(prepare) < steps.indexOf(faults)); + assert.doesNotMatch(prepare, /^\s+(?:if|continue-on-error):/m); +}); + test("Runner eval workflows pin actions and gate paid live execution", () => { const actionPinWorkflows = [ readWorkflow("release-verify.yml"),