diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index eca9ea71bfe..9ee40e8373c 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -1294,12 +1294,16 @@ jobs: fi exit "$status" - - name: Render the same canonical grid from a public-safe evidence projection + - name: Project safe run records for reporting if: needs.authorize.outputs.accounting_mode == 'legacy' || needs.authorize.outputs.testops_enabled == 'true' run: | node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs sanitize \ --runs-root runner-protocol-merged/runs \ --output runner-protocol-merged/public-runs + + - name: Render the same canonical grid from a public-safe evidence projection + if: needs.authorize.outputs.accounting_mode == 'legacy' + run: | python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ --runs-root runner-protocol-merged/public-runs \ --output runner-protocol-merged/public-report \ @@ -1312,11 +1316,24 @@ jobs: - name: Create the closed TestOps projection inside the report boundary if: always() && !cancelled() && needs.authorize.outputs.testops_enabled == 'true' && needs.testops_selection.result == 'success' id: testops_projection + env: + ACCOUNTING_MODE: ${{ needs.authorize.outputs.accounting_mode }} run: | mkdir -m 700 runner-protocol-testops-projection - node packages/paperclip-runner/scripts/runner-protocol-testops.mjs export \ - runner-protocol-testops-selection/runner-protocol-testops-selection.json \ - runner-protocol-merged/public-report runner-protocol-testops-projection/results + case "$ACCOUNTING_MODE" in + controlled) + node packages/paperclip-runner/scripts/runner-protocol-testops.mjs export-runs \ + runner-protocol-testops-selection/runner-protocol-testops-selection.json \ + runner-protocol-merged/campaign.json runner-protocol-merged/public-runs \ + runner-protocol-testops-projection/results + ;; + legacy) + node packages/paperclip-runner/scripts/runner-protocol-testops.mjs export \ + runner-protocol-testops-selection/runner-protocol-testops-selection.json \ + runner-protocol-merged/public-report runner-protocol-testops-projection/results + ;; + *) exit 1 ;; + esac - name: Transfer only the closed TestOps results and receipt if: always() && steps.testops_projection.outcome == 'success' diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index dece4dc0e67..55de332004d 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -1017,9 +1017,13 @@ A TestOps-triggered job uses its requested selection. Targeted selections retain `full-run` launch tag. Full selections use `full-run` and `test-plan-3`. The report job creates a closed projection of case identities, checks, numeric -usage and statuses. For controlled mode this happens inside the existing -private-history boundary; raw attempts, controller evidence and chat reports -remain there. Only the closed Allure results and receipt cross into the final +usage and statuses. For controlled mode, `runner-protocol-testops.mjs export-runs` +reads sanitized structured score/artifact records inside the existing +private-history boundary. It binds the original grade, complete checks, +attempt, model, provider and driver before it can emit a pass. Raw attempts, +controller evidence and canonical chat reports remain private. Controlled +reporting never invokes the public viewer; legacy publication keeps that path. +Only the closed Allure results and receipt cross into the final protected reporting job. It checks source and attempt identity, uploads one result per selected case, and verifies the saved case and status bindings. Missing execution/report evidence becomes a broken result. A passing campaign row diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index dff8204861f..6694faf3781 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/materialize-pi-distribution.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/acpx-release-attestations.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-kimi-distribution.test.mjs scripts/provision-kimi-package.test.mjs scripts/kimi-metadata-probe.test.mjs scripts/patch-kimi-runtime.test.mjs test/kimi-installed-runtime.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-provider-node.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/contracts/contract-fingerprints.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/materialize-pi-distribution.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/acpx-release-attestations.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-kimi-distribution.test.mjs scripts/provision-kimi-package.test.mjs scripts/kimi-metadata-probe.test.mjs scripts/patch-kimi-runtime.test.mjs test/kimi-installed-runtime.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-provider-node.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/contracts/contract-fingerprints.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs scripts/runner-protocol-testops.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index b15dc628f36..dae4647342f 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -406,6 +406,15 @@ test('TestOps receives only a closed projection created inside the report bounda assert.equal(admits(false, true, false), false); assert.match(report, /environment:.*runner-e2e-private-history/u); assert.match(report, /Create the closed TestOps projection inside the report boundary[\s\S]*runner-protocol-testops\.mjs export/u); + const publicGrid = report.slice(report.indexOf(' - name: Render the same canonical grid'), report.indexOf(' - name: Create the closed TestOps projection')); + assert.match(publicGrid, /if: needs\.authorize\.outputs\.accounting_mode == 'legacy'/u); + assert.doesNotMatch(publicGrid, /testops_enabled|env -u|unset /u); + const sanitize = report.slice(report.indexOf(' - name: Project safe run records'), report.indexOf(' - name: Render the same canonical grid')); + assert.match(sanitize, /accounting_mode == 'legacy' \|\| needs\.authorize\.outputs\.testops_enabled == 'true'/u); + assert.match(sanitize, /runner-protocol-eval-campaign\.mjs sanitize/u); + const projection = workflowShell(report, 'Create the closed TestOps projection inside the report boundary'); + assert.match(projection, /controlled\)[\s\S]*export-runs[\s\S]*campaign\.json runner-protocol-merged\/public-runs/u); + assert.match(projection, /legacy\)[\s\S]*export \\[\s\S]*runner-protocol-merged\/public-report/u); assert.match(report, /Transfer only the closed TestOps results and receipt[\s\S]*path: runner-protocol-testops-projection\//u); assert.match(publisher, /name: runner-protocol-testops-projection-/u); assert.doesNotMatch(publisher, /runner-protocol-eval-public-|runner-protocol-merged|private-history|AWS_ACCESS_KEY_ID|BRAINTRUST_API_KEY/u); diff --git a/packages/paperclip-runner/scripts/runner-protocol-testops.mjs b/packages/paperclip-runner/scripts/runner-protocol-testops.mjs index 79d37aab983..6f4eedef8bb 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-testops.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-testops.mjs @@ -139,6 +139,15 @@ export function filterCatalog(catalog, selection) { caseIds: [...selectedIds], allureJobRunId: selection.allureJobRunId } }; } +export function payloadFromSanitizedRun(score, artifact, cell) { + if (score?.schema !== "paperclip-runner/eval-score/v1" || artifact?.schema !== "paperclip-runner/eval-session-artifact/v1" + || score.attemptId !== cell.finalAttemptId || artifact.attemptId !== cell.finalAttemptId || score.caseId !== cell.caseId + || typeof score.passed !== "boolean" || artifact.publication?.schema !== "paperclip.runner-protocol-eval.public-chat/v1" + || artifact.requestedModel !== contract.model || artifact.provider !== contract.provider || artifact.driver !== contract.driver) fail("sanitized_payload_binding"); + return { attemptId: score.attemptId, caseId: score.caseId, disposition: score.disposition, passed: score.passed, + publication: { schema: artifact.publication.schema }, checks: score.checks }; +} + export function projectResults({ selection, campaign, payloads = new Map() }) { validateSelection(selection); const campaignUrl = runUrl(selection.runId); @@ -244,27 +253,36 @@ async function main() { await githubOutputs({ matrix_0: JSON.stringify(catalog.matrices[0]), matrix_1: JSON.stringify(catalog.matrices[1]), matrix_1_present: String(catalog.matrices[1].include.length > 0), selected: catalog.cells.length }); console.log(json({ selected: catalog.cells.length, fullPlan: catalog.selection.fullPlan })); - } else if (command === "export") { + } else if (command === "export" || command === "export-runs") { + const structured = command === "export-runs"; + if (args.length !== (structured ? 4 : 3)) fail("arguments"); const selection = await readJson(args[0]); + const output = args[structured ? 3 : 2]; let campaign; - try { campaign = await readJson(join(args[1], "campaign.json")); } catch (error) { if (error.code !== "ENOENT") throw error; } + try { campaign = await readJson(structured ? args[1] : join(args[1], "campaign.json")); } catch (error) { if (error.code !== "ENOENT") throw error; } const payloads = new Map(); for (const row of campaign?.results ?? []) { if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(row.finalAttemptId)) fail("attempt_path"); try { - const html = await readFile(join(args[1], "attempts", row.finalAttemptId, "index.html"), "utf8"); - const match = /