diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 9ee40e8373c..f2528e850aa 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -71,7 +71,6 @@ jobs: testops_enabled: ${{ steps.mode.outputs.testops_enabled }} accounting_mode: ${{ steps.mode.outputs.mode }} retry_count: ${{ steps.mode.outputs.retries }} - controller_lock_sha256: ${{ steps.mode.outputs.controller_lock_sha256 }} test_runner: ${{ steps.runner.outputs.runner }} max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} @@ -168,7 +167,6 @@ jobs: TESTOPS_REQUESTED: ${{ inputs.report_to_testops || inputs.ALLURE_JOB_RUN_ID != '' }} REQUESTED_ROSTERS: ${{ inputs.rosters }} RETRIES: ${{ inputs.max_infrastructure_retries || 0 }} - CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} TARGET_SHA: ${{ steps.target.outputs.sha }} run: | set -euo pipefail @@ -179,12 +177,6 @@ jobs: test "$TARGET_SHA" = "$GITHUB_SHA" test "$GITHUB_RUN_ATTEMPT" = 1 test "$RETRIES" = 0 - if ! jq -e '.schema == "paperclip.runner-eval.hosted-policy/v1"' <<< "${CONTROLLER_POLICY_JSON:-}" >/dev/null; then - echo 'Controlled runs require a reviewed RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON; no provider work started.' >&2 - exit 1 - fi - lock_sha="$(jq -er '.controllerLockSha256 | select(type == "string" and test("^[a-f0-9]{64}$"))' <<< "$CONTROLLER_POLICY_JSON")" - echo "controller_lock_sha256=$lock_sha" >> "$GITHUB_OUTPUT" ;; legacy) test "$EVENT_NAME" = workflow_dispatch ;; *) exit 1 ;; @@ -415,15 +407,19 @@ jobs: - name: Enforce controlled limits against the selected catalog if: needs.authorize.outputs.accounting_mode == 'controlled' - env: - CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} run: | set -euo pipefail jq -e 'all(.cells[]; (.provider == "codex" and (.acpxAgent // "") == "" and .driver == "codex_app_server") or (.provider == "acpx" and .acpxAgent == "claude" and .driver == "acpx_runtime"))' runner-protocol-eval-catalog.json >/dev/null || { echo 'Controlled selection supports native Codex and ACPX Claude only; select their rosters explicitly.' >&2; exit 1; } - maximum="$(jq -er '.maxCells | select(type == "number" and floor == . and . > 0 and . <= 512)' <<< "$CONTROLLER_POLICY_JSON")" - jq -e --argjson maximum "$maximum" '.cells | length <= $maximum' runner-protocol-eval-catalog.json >/dev/null + # Every selected profile needs a committed allocation. No evals code has run in this job yet. + jq -e --slurpfile defaults packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json \ + '($defaults[0].allocations | map([.provider, (.acpxAgent // ""), .model])) as $allowed | all(.cells[]; [.provider, (.acpxAgent // ""), .model] as $profile | any($allowed[]; . == $profile))' \ + runner-protocol-eval-catalog.json >/dev/null || { + echo 'Controlled selection includes a profile without a committed allocation in hosted-defaults.json.' >&2; exit 1; + } + # The campaign size is the selected catalog; the spend cap is enforced when the manifest is prepared. + jq -e '(.cells | length) >= 1 and (.cells | length) <= 512' runner-protocol-eval-catalog.json >/dev/null - name: Require the chat-report renderer before paid execution run: | @@ -549,7 +545,7 @@ jobs: build_controller: name: Build trusted controller without private archive access - needs: [authorize, target_lock] + needs: [authorize] runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -579,26 +575,14 @@ jobs: with: version: 9.15.4 - - name: Download resolved lockfile for review binding - if: needs.authorize.outputs.accounting_mode == 'controlled' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} - path: ${{ runner.temp }}/runner-controller-lock - - name: Build portable controller in an unprivileged job if: needs.authorize.outputs.accounting_mode == 'controlled' id: build - env: - EXPECTED_LOCK_SHA256: ${{ needs.authorize.outputs.controller_lock_sha256 }} - RESOLVED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - test "$RESOLVED_LOCK_SHA256" = "$EXPECTED_LOCK_SHA256" - lock="$RUNNER_TEMP/runner-controller-lock/pnpm-lock.yaml" - test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" - cp "$lock" pnpm-lock.yaml + # The controller installs exactly the committed lockfile of the trusted revision. + git diff --quiet HEAD -- pnpm-lock.yaml pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/runner-controller-pnpm-store" node packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs \ "$RUNNER_TEMP/runner-protocol-runtime" "$GITHUB_SHA" > "$RUNNER_TEMP/controller-runtime-receipt.json" @@ -700,21 +684,27 @@ jobs: - name: Verify copied controller before private preparation if: needs.authorize.outputs.accounting_mode == 'controlled' + id: controller_lock env: - EXPECTED_LOCK_SHA256: ${{ needs.authorize.outputs.controller_lock_sha256 }} RUNTIME_RECEIPT_SHA256: ${{ needs.build_controller.outputs.runtime_receipt_sha256 }} run: | set -euo pipefail node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs "$RUNNER_TEMP/runner-protocol-runtime" "$GITHUB_SHA" "$RUNTIME_RECEIPT_SHA256" - test "$(jq -r '.inputs["pnpm-lock.yaml"]' "$RUNNER_TEMP/runner-protocol-runtime/runtime.json")" = "$EXPECTED_LOCK_SHA256" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + committed="$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" + test "$(jq -r '.inputs["pnpm-lock.yaml"]' "$RUNNER_TEMP/runner-protocol-runtime/runtime.json")" = "$committed" + echo "sha256=$committed" >> "$GITHUB_OUTPUT" - - name: Compile requests and bind the reviewed operating policy + - name: Compile requests and derive the committed operating policy if: needs.authorize.outputs.accounting_mode == 'controlled' id: prepare env: PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} - CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} + PAPERCLIP_RUNNER_EVAL_CONTROLLER_LOCK_SHA256: ${{ steps.controller_lock.outputs.sha256 }} + PAPERCLIP_RUNNER_EVAL_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }} + # Optional campaign spend cap; the committed default applies when unset. + RUNNER_PROTOCOL_EVAL_MAX_USD: ${{ vars.RUNNER_PROTOCOL_EVAL_MAX_USD }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$GITHUB_SHA" @@ -723,12 +713,6 @@ jobs: (cd runner-protocol-build && sha256sum --check runner-protocol-build.tar.gz.sha256) mkdir runner-protocol-build/extracted tar -xzf runner-protocol-build/runner-protocol-build.tar.gz -C runner-protocol-build/extracted - policy="$RUNNER_TEMP/runner-controller-policy.json" - POLICY_PATH="$policy" python3 - <<'PYTHON' - import os - with os.fdopen(os.open(os.environ["POLICY_PATH"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as target: - target.write(os.environ["CONTROLLER_POLICY_JSON"]) - PYTHON worker="/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}" python3 .paperclip-evals/evals/paperclip-runner/tools/prepare_controlled_campaign.py \ --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \ @@ -739,8 +723,7 @@ jobs: control="$RUNNER_TEMP/runner-protocol-control" node "$RUNNER_TEMP/runner-protocol-runtime/controlled-evals/hosted.mjs" prepare \ runner-protocol-catalog/runner-protocol-eval-catalog.json "$RUNNER_TEMP/prepared-requests.json" \ - "$policy" runner-protocol-build/extracted "$control" > "$RUNNER_TEMP/control-preparation.json" - cp "$policy" "$control/hosted-policy.json" + runner-protocol-build/extracted "$control" > "$RUNNER_TEMP/control-preparation.json" echo "manifest_sha256=$(jq -r .manifestSha256 "$RUNNER_TEMP/control-preparation.json")" >> "$GITHUB_OUTPUT" - name: Obtain private archive credentials for transfer steps only diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 55de332004d..4aba2c62d48 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -550,7 +550,9 @@ from the default branch and provide: selection; `all` never spends against a lane that the eval program has marked disabled; - `accounting_mode`: `controlled` by default; `legacy` is an explicit manual - qualification path. Controlled mode supports native Codex and ACPX Claude. + qualification path. Controlled mode supports the profiles that have a + committed allocation in `hosted-defaults.json`; today that is ACPX Claude + Sonnet 5. Native Codex also needs a committed OpenAI price and allocation. - `max_infrastructure_retries`: zero by default and required for controlled mode. Legacy mode allows up to three retries only when an attempt explicitly reports a retryable infrastructure failure. @@ -843,9 +845,10 @@ forced stop still leaves cleanup unverified and retains the full allocation. ## Controlled hosted workflow `accounting_mode=controlled` is the default for manual dispatch and the only -scheduled mode. It supports explicitly selected native Codex and ACPX Claude -rosters. `all` still means the entire maintained suite; unsupported cells fail -admission rather than being omitted. Use `accounting_mode=legacy` explicitly +scheduled mode. It supports explicitly selected rosters whose profiles have a +committed allocation, today ACPX Claude Sonnet 5. `all` still means the entire +maintained suite; unsupported cells fail the catalog step before any build +rather than being omitted. Use `accounting_mode=legacy` explicitly for a manual qualification of other providers. Legacy runs retain their existing public artifact and publication behavior and do not claim complete controller cost coverage. @@ -858,32 +861,49 @@ Re-running even one job is rejected at that worker before provider access. The existing nightly enable gate and exact eval SHA pin remain required. This change does not set either variable or enable paid recurrence. -Set `RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON` only after its operating -bounds have been reviewed. An absent, stale or invalid policy stops before -provider work. The policy schema is `paperclip.runner-eval.hosted-policy/v1`: +Controlled runs derive their campaign policy from committed defaults in +`packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json`. +Changing a price, allocation, overhead reservation or trace limit is a reviewed +pull request, not a repository variable. The controller runtime packages the +file, and `hosted.mjs prepare` reads that verified copy, never the workspace +that the private evals program can modify. The file holds: -- `controllerLockSha256`, the SHA-256 of the explicitly reviewed compatible - workspace lockfile. The existing unprivileged target-lock job resolves it; - inspect that immutable artifact before setting the policy. The controller - build requires its digest to match, installs with `--frozen-lockfile` and a - job-local store, and records the digest in the runtime receipt. Private - preparation checks that receipt binding again. A changed lockfile requires a - policy review. No lockfile is committed by this workflow integration. -- `maxUsd`, `maxCells` (1–512), `durationMs` (at most four hours), current - `pricingVerifiedAt` and `pricingExpiresAt`. -- `route`, optional gateway `projectId`, exact `models`, and the same - four-component `overhead` contract used by the controller. -- `observability` with `projectId`, `capture` and `nativeLimits`. Tracing is - required for this mode. `BRAINTRUST_API_KEY` is delivered only to the - trusted controller step, even when model requests use the direct route. +- `maxUsd`, the default campaign spend cap. The optional repository variable + `RUNNER_PROTOCOL_EVAL_MAX_USD` overrides it; it is the only setting. +- `durationMs` (at most four hours) and `route`. +- `models`, the committed price table. Prices carry a version and source URL + and no review expiry. Each campaign receives a pricing and overhead validity + window that starts when its manifest is prepared and ends one hour after its + deadline. - `allocations`, one entry per provider/agent/model profile, each with `maxUsd`, `maxRequests`, `maxConcurrent` and `timeoutMs`. -- `capacityBound`, a reviewed full-lifecycle proof. Its `configurationSha256` - hashes `canonical(hostedPolicyConfiguration(policy))`, which excludes only - the proof itself. The proof must cover every selected cell up to `maxCells`, - trace and gateway processing, retained data, private archive transfers, - workflow setup/build/reporting and the full retention window. A policy - digest is a binding to evidence, not independent proof of vendor charges. +- `overhead`, the four-component operating reservation with + `accountingReadiness: "model_requests"`. +- `observability` capture flags and `nativeLimits`. Tracing is required. The + Braintrust project comes from `RUNNER_E2E_BRAINTRUST_PROJECT_ID`. + `BRAINTRUST_API_KEY` is delivered only to the trusted controller step, even + when model requests use the direct route. + +The cell limit is the selected catalog itself (1–512 cells). The campaign must +still fit its spend cap: the once-only overhead plus every selected allocation +must not exceed `maxUsd`, or preparation fails before provider work. With the +committed defaults the 41-case Claude plan reserves $339 of the $400 cap. + +The controller installs the committed `pnpm-lock.yaml` of the trusted workflow +revision with `--frozen-lockfile`, disabled install scripts and a job-local +store. It fails if the checkout's lockfile differs from the commit. The runtime +receipt records the lockfile digest; private preparation checks that it equals +the committed lockfile before compiling requests. A dependency change is +reviewed through its own pull request. + +The previous `RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON` variable is no longer +read. Its hand-maintained `maxCells`, pricing windows, `controllerLockSha256` +and self-hashed `capacityBound`/`provisionalBound` proofs are retired. Under +model-request readiness every request settles from its own receipt, and an +unresolved request fails the cell, so those proofs never changed admission or +accounting. Retained manifests that include them still replay. The derived +policy is retained next to the manifest as `hosted-policy.json` in private +storage. The preparer compiles every selected private request once. It validates exact catalog membership, source and definition revisions, executable digests and @@ -917,7 +937,7 @@ role, region and bucket variables. The role and bucket policy are unchanged. The transfer helper verifies bucket privacy and 30-day bounded retention, rejects unsafe archives, uses conditional immutable writes and does not retry. Each archive is at most 264 MiB (256 MiB source). The reviewed operating bound -must cover up to `maxCells` cell archives, one controller archive, one report +must cover every selected cell archive, one controller archive, one report archive, their download/receipt traffic and the GitHub code/build artifacts. Each shared manifest downloader supplies its worker cell ID or `report` as a receipt identity. Receipts use distinct immutable keys while all workers @@ -962,13 +982,12 @@ paid recurrence disabled until these independent gates are qualified. ### Token-spend qualification with separate operating reconciliation -A policy may explicitly set `overhead.accountingReadiness: "model_requests"` -when model-token qualification is the operator's priority. In that mode an -absent campaign capacity proof and absent observability cost proof do not block -admission. Any supplied proofs must still validate. Current allowance/pricing -windows, trace quotas, immutable source/request/runtime bindings, complete -selection, per-cell allocations, request limits, no retries and the shared -campaign admission cap remain required. Model reservation proofs remain required. +The committed hosted defaults use `overhead.accountingReadiness: "model_requests"`. +In that mode absent capacity, observability and model reservation proofs do not +block admission; any supplied proofs must still validate. Trace quotas, +immutable source/request/runtime bindings, complete selection, per-cell +allocations, request limits, no retries and the campaign spend cap remain +required. The final report retains the once-only operating allowance and labels operating reconciliation pending. Missing, failed or canceled model evidence still holds @@ -995,7 +1014,7 @@ The existing workflow accepts the optional `ALLURE_JOB_RUN_ID` and roster, and an explicit `accounting_mode` for the standing 41-case protocol plan. The source must be the trusted current workflow revision. Qualification covers that recorded Claude Sonnet 5 configuration and accounting mode. -Controlled mode preserves its reviewed policy, immutable lock/source/request +Controlled mode preserves its committed policy, immutable lock/source/request bindings, tracing, private retention, first-attempt and zero-retry gates. Its paid workers use `ubuntu-latest`, matching the required GitHub-hosted operating origin even when the legacy AWS fleet selector is enabled. diff --git a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs index c58f6e23b74..20670f05fe9 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs @@ -10,7 +10,7 @@ const source = dirname(fileURLToPath(import.meta.url)); const repository = resolve(source, '../../../..'); const require = createRequire(join(repository, 'server/package.json')); const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); -const scripts = ['hosted.mjs', 'entrypoint.mjs', 'evidence-files.mjs', 'archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; +const scripts = ['hosted.mjs', 'hosted-defaults.json', 'entrypoint.mjs', 'evidence-files.mjs', 'archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; export async function buildControllerRuntime({ outputDirectory, sourceRevision }) { if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error('Controller runtime requires an exact source revision'); diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json b/packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json new file mode 100644 index 00000000000..a2625ce3edb --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json @@ -0,0 +1,59 @@ +{ + "schema": "paperclip.runner-eval.hosted-defaults/v1", + "maxUsd": "400", + "durationMs": 14400000, + "route": "braintrust", + "models": [ + { + "provider": "anthropic", + "maximumInputTokens": 1000000, + "maximumOutputTokens": 16384, + "timeoutMs": 180000, + "price": { + "provider": "anthropic", + "model": "claude-sonnet-5", + "version": "anthropic-standard-2026-11-04", + "sourceUrl": "https://platform.claude.com/docs/en/models/sonnet-5/overview", + "inputUsdPerMillion": "2", + "cachedReadUsdPerMillion": "0.2", + "cacheWrite5mUsdPerMillion": "2.5", + "cacheWrite1hUsdPerMillion": "4", + "outputUsdPerMillion": "10" + } + } + ], + "allocations": [ + { + "provider": "acpx", + "acpxAgent": "claude", + "model": "claude-sonnet-5", + "maxUsd": "8", + "maxRequests": 20, + "maxConcurrent": 1, + "timeoutMs": 600000 + } + ], + "overhead": { + "version": 1, + "accountingReadiness": "model_requests", + "reference": "committed-hosted-defaults", + "runner": "github-hosted", + "components": [ + { "component": "ci_compute", "maxUsd": "1", "reference": "operator-github-hosted-reservation-not-invoice" }, + { "component": "artifact_storage", "maxUsd": "1", "reference": "operator-private-history-reservation-not-invoice" }, + { "component": "observability", "maxUsd": "9", "reference": "operator-reservation-pending-vendor-reconciliation" }, + { "component": "shared_subscription", "maxUsd": "0", "reference": "existing-subscription-shown-separately" } + ] + }, + "observability": { + "capture": { "assistantResponses": false, "toolContent": false, "toolDetails": false, "userPrompts": false }, + "nativeLimits": { + "maxConcurrent": 1, + "maxExpandedBytes": 16777216, + "maxRecords": 4096, + "maxRequestBytes": 1048576, + "maxRequests": 64, + "maxTotalBytes": 16777216 + } + } +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs index 7a6f15dcc55..449871613a0 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs @@ -1,7 +1,8 @@ import { readFile, mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { isEntrypoint } from './entrypoint.mjs'; -import { assertHostedAdmission, canonical, costRuntime, fields, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; +import { assertHostedAdmission, canonical, fields, requestBindingSha256, sha256, units, validateManifest } from './manifest.mjs'; import { readRegularFile } from './evidence-files.mjs'; export function hostedPaths(runId) { @@ -11,21 +12,41 @@ export function hostedPaths(runId) { manifest: join(root, 'controller-manifest.json'), evidence: join(root, 'evidence') }; } -export function hostedPolicyConfiguration(policy) { - const { capacityBound, ...configuration } = policy; - return configuration; +export const HOSTED_DEFAULTS_SCHEMA = 'paperclip.runner-eval.hosted-defaults/v1'; +export const HOSTED_POLICY_SCHEMA = 'paperclip.runner-eval.hosted-policy/v2'; +const MAX_HOSTED_CELLS = 512; +// The committed price table has no review expiry. Each campaign receives a +// validity window that covers only its own deadline plus a short settlement margin. +const SETTLEMENT_MARGIN_MS = 3_600_000; + +// Derive the campaign policy from committed defaults. Cell count comes from the +// selected catalog, prices from the committed table and the dependency lock from +// the committed frozen lockfile. Only the campaign spend cap may be overridden. +export function hostedPolicy(defaults, { cellCount, now = Date.now(), maxUsd, projectId, controllerLockSha256 }) { + fields(defaults, ['schema', 'maxUsd', 'durationMs', 'route', 'models', 'allocations', 'overhead', 'observability']); + if (defaults.schema !== HOSTED_DEFAULTS_SCHEMA || !defaults.observability || !Array.isArray(defaults.models) || !defaults.models.length + || !Number.isSafeInteger(defaults.durationMs) || defaults.durationMs < 1 || defaults.durationMs > 4 * 3_600_000 + || !Array.isArray(defaults.allocations) || !defaults.allocations.length) throw new Error('A bounded hosted policy with tracing is required'); + if (!Number.isSafeInteger(cellCount) || cellCount < 1 || cellCount > MAX_HOSTED_CELLS) throw new Error('Invalid controlled campaign selection'); + if (!/^[a-f0-9]{64}$/.test(controllerLockSha256 ?? '')) throw new Error('Invalid hosted controller lock binding'); + fields(defaults.overhead, ['version', 'accountingReadiness', 'reference', 'runner', 'components']); + fields(defaults.observability, ['capture', 'nativeLimits']); + if (defaults.overhead.accountingReadiness !== 'model_requests') throw new Error('Controller requires current full-lifecycle operating bounds'); + const cap = maxUsd === undefined || maxUsd === '' ? defaults.maxUsd : maxUsd; + units(cap); + const verifiedAt = new Date(now).toISOString(); + const expiresAt = new Date(now + defaults.durationMs + SETTLEMENT_MARGIN_MS).toISOString(); + return { schema: HOSTED_POLICY_SCHEMA, maxUsd: cap, durationMs: defaults.durationMs, maxCells: cellCount, + pricingVerifiedAt: verifiedAt, pricingExpiresAt: expiresAt, route: defaults.route, + ...(defaults.route === 'braintrust' && projectId ? { projectId } : {}), + models: structuredClone(defaults.models), allocations: structuredClone(defaults.allocations), + overhead: { ...structuredClone(defaults.overhead), verifiedAt, expiresAt }, + observability: { ...(projectId ? { projectId } : {}), ...structuredClone(defaults.observability) }, controllerLockSha256 }; } -export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy, artifacts, environment, now = Date.now() }) { - fields(policy, ['schema', 'maxUsd', 'durationMs', 'pricingVerifiedAt', 'pricingExpiresAt', 'route', 'projectId', 'models', 'overhead', 'observability', 'allocations', 'maxCells', 'capacityBound', 'controllerLockSha256']); - if (policy.schema !== 'paperclip.runner-eval.hosted-policy/v1' || !policy.observability - || !Number.isSafeInteger(policy.durationMs) || policy.durationMs < 1 || policy.durationMs > 4 * 3_600_000 - || !Number.isSafeInteger(policy.maxCells) || policy.maxCells < 1 || policy.maxCells > 512 - || !Array.isArray(policy.allocations) || !policy.allocations.length) throw new Error('A bounded hosted policy with tracing is required'); - if (!/^[a-f0-9]{64}$/.test(policy.controllerLockSha256 ?? '')) throw new Error('Invalid hosted controller lock binding'); - if (policy.overhead?.accountingReadiness !== 'model_requests' || policy.capacityBound !== undefined) { - costRuntime.reservationBound('operating_allowance', policy.capacityBound, hostedPolicyConfiguration(policy)); - } +export function prepareHostedManifest({ catalog, catalogSha256, prepared, defaults, maxUsd, projectId, controllerLockSha256, artifacts, environment, now = Date.now() }) { + const policy = hostedPolicy(defaults, { cellCount: Array.isArray(prepared?.cells) ? prepared.cells.length : 0, + now, maxUsd, projectId, controllerLockSha256 }); fields(policy.observability, ['projectId', 'capture', 'nativeLimits']); const runId = environment.GITHUB_RUN_ID, paths = hostedPaths(runId); if (prepared.schema !== 'paperclip.runner-eval.prepared-requests/v1' || prepared.githubRunId !== runId || prepared.githubRunAttempt !== 1 @@ -67,10 +88,9 @@ export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy models: policy.models, overhead: policy.overhead, observability: { ...policy.observability }, sourceRevision: prepared.sourceRevision, definitionsRevision: prepared.definitionsRevision, ...artifacts, outputDirectory: paths.evidence, hosted: { repository: environment.GITHUB_REPOSITORY, runId, runAttempt: 1, workflowRevision: environment.GITHUB_SHA, controllerLockSha256: policy.controllerLockSha256, policySha256: sha256(canonical(policy)) }, attempts }; - if (policy.capacityBound !== undefined) manifest.observability.provisionalBound = { ...policy.capacityBound, configurationSha256: sha256(canonical(observabilityBoundConfiguration(manifest))) }; const validated = validateManifest(manifest, now); assertHostedAdmission(manifest, attempts[0], { ...environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: attempts[0].executionId }); - return validated; + return { ...validated, policy }; } export async function verifyHostedCell({ manifestPath, manifestSha256, cellId, environment }) { @@ -101,14 +121,17 @@ export function hostedAdmissionDiagnostic(error) { if (error instanceof Error && safeAdmissionReasons.has(error.message)) return error.message; if (error?.code === 'ENOENT') return 'A required hosted input file is missing'; if (error instanceof SyntaxError) return 'A hosted input is not valid JSON'; - return 'Invalid or unreadable hosted input; check the catalog, policy and runtime bindings'; + return 'Invalid or unreadable hosted input; check the catalog, committed defaults and runtime bindings'; } if (await isEntrypoint(import.meta.url)) { try { const [command, ...args] = process.argv.slice(2); - if (command === 'prepare' && args.length === 5) { - const [catalogPath, preparedPath, policyPath, runner, output] = args; + if (command === 'prepare' && args.length === 4) { + // Defaults are read beside this verified controller code, never from the workspace, + // which the private evals program can modify during preparation. + const [catalogPath, preparedPath, runner, output] = args; + const defaultsPath = fileURLToPath(new URL('./hosted-defaults.json', import.meta.url)); const json = async path => JSON.parse((await readRegularFile(path, 16 * 1_048_576)).toString('utf8')); const catalogBytes = await readRegularFile(catalogPath, 16 * 1_048_576); const prepared = await json(preparedPath), paths = hostedPaths(process.env.GITHUB_RUN_ID), artifacts = {}; @@ -116,14 +139,17 @@ if (await isEntrypoint(import.meta.url)) { artifacts[name] = { path: join(paths.runner, relative), sha256: sha256(await readFile(join(runner, relative))) }; } const result = prepareHostedManifest({ catalog: JSON.parse(catalogBytes), catalogSha256: sha256(catalogBytes), prepared, - policy: await json(policyPath), artifacts, environment: process.env }); + defaults: await json(defaultsPath), maxUsd: process.env.RUNNER_PROTOCOL_EVAL_MAX_USD || undefined, + projectId: process.env.PAPERCLIP_RUNNER_EVAL_BRAINTRUST_PROJECT_ID || undefined, + controllerLockSha256: process.env.PAPERCLIP_RUNNER_EVAL_CONTROLLER_LOCK_SHA256, artifacts, environment: process.env }); await mkdir(output, { mode: 0o700 }); await writeFile(join(output, 'controller-manifest.json'), canonical(result.manifest), { flag: 'wx', mode: 0o600 }); + await writeFile(join(output, 'hosted-policy.json'), `${JSON.stringify(result.policy, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); console.log(JSON.stringify({ manifestSha256: result.manifestSha256, cells: result.manifest.attempts.length, - allocatedUsd: result.allocatedUsd, providerCalls: 0 })); + maxUsd: result.manifest.maxUsd, allocatedUsd: result.allocatedUsd, providerCalls: 0 })); } else if (command === 'verify' && args.length === 3) { await verifyHostedCell({ manifestPath: args[0], manifestSha256: args[1], cellId: args[2], environment: process.env }); - } else throw new Error('Usage: hosted.mjs '); + } else throw new Error('Usage: hosted.mjs '); } catch (error) { console.error(`Hosted campaign admission failed: ${hostedAdmissionDiagnostic(error)}. No provider work was started by this command.`); process.exitCode = 2; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs index 89b14b2bb39..7be1556b99d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs @@ -6,9 +6,9 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { fixture, success } from './controller-fixture.mjs'; -import { canonical, sha256, validateManifest } from './manifest.mjs'; +import { canonical, sha256, units, validateManifest } from './manifest.mjs'; const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; -const { hostedPaths, hostedPolicyConfiguration, prepareHostedManifest, verifyHostedCell, hostedAdmissionDiagnostic } = await import(new URL('./hosted.mjs', runtime)); +const { hostedPaths, hostedPolicy, prepareHostedManifest, verifyHostedCell, hostedAdmissionDiagnostic } = await import(new URL('./hosted.mjs', runtime)); const { packControlledAttempt, summarizeControlledManifest } = await import(new URL('./archive.mjs', runtime)); async function campaign(t) { @@ -30,16 +30,17 @@ async function campaign(t) { const prepared = { schema: 'paperclip.runner-eval.prepared-requests/v1', campaignId: catalog.campaignId, githubRunId: runId, githubRunAttempt: 1, sourceRevision: f.manifest.sourceRevision, definitionsRevision: f.manifest.definitionsRevision, catalogSha256, runnerd: f.requests[0].request.runnerd, cells: cells.map((cell, index) => ({ cellId: cell.cellId, runId: `${catalog.campaignId}-${cell.cellId}`, request: f.requests[index].request })) }; - for (const component of f.manifest.overhead.components) component.provisionalBound.evidenceSha256 = sha256(component.component); - const { provisionalBound, ...observability } = f.manifest.observability; + const { projectId, capture, nativeLimits } = f.manifest.observability; const { executionId, attemptId, requestSha256, ...allocation } = f.manifest.attempts[0]; - const policy = { schema: 'paperclip.runner-eval.hosted-policy/v1', controllerLockSha256: 'e'.repeat(64), maxUsd: f.manifest.maxUsd, maxCells: 2, durationMs: 60000, - pricingVerifiedAt: f.manifest.pricingVerifiedAt, pricingExpiresAt: f.manifest.pricingExpiresAt, - route: 'direct', models: f.manifest.models, overhead: f.manifest.overhead, observability, allocations: [allocation] }; - policy.capacityBound = { version: 1, scope: 'full_lifecycle', evidenceSha256: f.manifest.overhead.components.find(component => component.component === 'observability').provisionalBound.evidenceSha256, configurationSha256: sha256(canonical(hostedPolicyConfiguration(policy))) }; + const { verifiedAt, expiresAt, components, ...overhead } = f.manifest.overhead; + // Committed defaults carry no cell count, review window, lock pin or self-hashed bounds. + const defaults = { schema: 'paperclip.runner-eval.hosted-defaults/v1', maxUsd: f.manifest.maxUsd, durationMs: 60000, route: 'direct', + models: f.manifest.models.map(({ provisionalBound, ...model }) => model), allocations: [allocation], + overhead: { ...overhead, accountingReadiness: 'model_requests', components: components.map(({ provisionalBound, ...component }) => component) }, + observability: { capture, nativeLimits } }; const environment = { ...f.environment, GITHUB_ACTIONS: 'true', GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_RUN_ID: runId, GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: f.manifest.sourceRevision, PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: prepared.sourceRevision, PAPERCLIP_PROTOCOL_EVALS_SHA: prepared.definitionsRevision }; - const input = { catalog, catalogSha256, prepared, policy, artifacts, environment }; + const input = { catalog, catalogSha256, prepared, defaults, projectId, controllerLockSha256: 'e'.repeat(64), artifacts, environment }; const build = () => prepareHostedManifest(input); return { f, input, paths, environment, build }; } @@ -76,34 +77,31 @@ test('hosted preparation rejects selection drift, unsupported cells and unreview input => { input.prepared.cells[0].request.driver = 'unsupported'; }, input => { input.catalog.cells[0].driver = 'unsupported'; input.catalog.matrices[0].include[0].driver = 'unsupported'; }, input => { input.prepared.cells[0].request.runnerd.sha256 = 'sha256:' + '0'.repeat(64); }, - input => { input.policy.maxUsd = '0.01'; }, + input => { input.maxUsd = '0.01'; }, + input => { input.maxUsd = 'unbounded'; }, + input => { input.prepared.cells = []; }, input => { input.environment.GITHUB_RUN_ATTEMPT = '2'; }, input => { input.environment.GITHUB_SHA = 'wrong'; }, input => { input.environment.GITHUB_SHA = 'c'.repeat(40); }, - input => { delete input.policy.observability; }, - input => { delete input.policy.controllerLockSha256; }, - input => { input.policy.controllerLockSha256 = 'f'.repeat(64); }, + input => { delete input.defaults.observability; }, + input => { delete input.controllerLockSha256; }, + input => { input.controllerLockSha256 = 'not-a-digest'; }, + input => { input.defaults.durationMs = 5 * 3_600_000; }, + input => { input.defaults.schema = 'paperclip.runner-eval.hosted-policy/v1'; }, ]; for (const change of changes) { const input = structuredClone(c.input); change(input); assert.throws(() => prepareHostedManifest(input)); } - const mixed = structuredClone(c.input); - mixed.policy.capacityBound.evidenceSha256 = mixed.policy.overhead.components.find(component => component.component !== 'observability').provisionalBound.evidenceSha256; - assert.throws(() => prepareHostedManifest(mixed), /Observability capacity requires its operating bound evidence/); - const input = structuredClone(c.input); input.policy.maxUsd = '0.01'; - input.policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(input.policy))); + const input = structuredClone(c.input); input.maxUsd = '0.01'; assert.throws(() => prepareHostedManifest(input), /budget/); }); test('hosted admission rejects local and fleet origins while preserving retained manifests', async t => { const c = await campaign(t); - c.input.policy.overhead.accountingReadiness = 'model_requests'; - for (const component of c.input.policy.overhead.components) delete component.provisionalBound; - delete c.input.policy.capacityBound; const original = c.build().manifest; for (const runner of ['local', 'aws-fleet']) { - const input = structuredClone(c.input); input.policy.overhead.runner = runner; + const input = structuredClone(c.input); input.defaults.overhead.runner = runner; assert.throws(() => prepareHostedManifest(input), /GitHub-hosted operating origin/); const retained = structuredClone(original); retained.overhead.runner = runner; assert.equal(validateManifest(retained, Date.now(), { admission: false }).manifest.overhead.runner, runner); @@ -142,9 +140,6 @@ test('hosted CLI reports safe admission reasons without echoing private paths or test('model-readiness policy retains operating holds without requiring vendor cost proofs', async t => { const c = await campaign(t); - c.input.policy.overhead.accountingReadiness = 'model_requests'; - for (const component of c.input.policy.overhead.components) delete component.provisionalBound; - delete c.input.policy.capacityBound; const result = c.build(); assert.equal(result.manifest.overhead.accountingReadiness, 'model_requests'); assert.equal(result.manifest.observability.provisionalBound, undefined); @@ -163,27 +158,46 @@ test('model-readiness policy retains operating holds without requiring vendor co assert.equal(complete.operatingHeldUsd, '0.004000000'); assert.equal(complete.accountingReadiness, 'model_requests'); assert.equal(complete.operatingReconciliation, 'pending'); - delete c.input.policy.overhead.accountingReadiness; - assert.throws(() => c.build(), /cost bound|operating bound/); + delete c.input.defaults.overhead.accountingReadiness; + assert.throws(() => c.build(), /operating bound/); }); -test('model readiness still validates supplied operating and capacity proofs', async t => { - const c = await campaign(t), policy = c.input.policy; - policy.overhead.accountingReadiness = 'model_requests'; - for (const component of policy.overhead.components) { - component.provisionalBound.configurationSha256 = sha256(canonical({ accountingReadiness: 'model_requests', - reference: policy.overhead.reference, runner: policy.overhead.runner, verifiedAt: policy.overhead.verifiedAt, - expiresAt: policy.overhead.expiresAt, component: { component: component.component, maxUsd: component.maxUsd, reference: component.reference } })); - } - policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(policy))); - assert.equal(c.build().manifest.overhead.accountingReadiness, 'model_requests'); - const mismatch = structuredClone(c.input); mismatch.policy.capacityBound.evidenceSha256 = '0'.repeat(64); - assert.throws(() => prepareHostedManifest(mismatch), /Observability capacity requires its operating bound evidence/); - const stale = structuredClone(c.input); stale.policy.capacityBound.configurationSha256 = '0'.repeat(64); - assert.throws(() => prepareHostedManifest(stale), /Reviewed cost bound configuration changed/); - const absent = structuredClone(c.input); - for (const component of absent.policy.overhead.components) delete component.provisionalBound; - absent.policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(absent.policy))); - assert.throws(() => prepareHostedManifest(absent), /Observability capacity requires its operating bound evidence/); +test('the campaign policy is derived from the selection and committed defaults', async t => { + const c = await campaign(t), now = Date.now(), result = c.build(); + assert.equal(result.policy.maxCells, 2); + assert.equal(result.policy.schema, 'paperclip.runner-eval.hosted-policy/v2'); + assert.equal(result.manifest.hosted.controllerLockSha256, 'e'.repeat(64)); + assert.equal(result.manifest.models[0].provisionalBound, undefined); + // The validity window covers this campaign only; the committed table itself never expires. + assert.ok(Date.parse(result.manifest.pricingVerifiedAt) <= Date.now() && Date.parse(result.manifest.pricingVerifiedAt) >= now - 1000); + assert.ok(Date.parse(result.manifest.pricingExpiresAt) > Date.parse(result.manifest.deadlineAt)); + assert.equal(result.manifest.overhead.expiresAt, result.manifest.pricingExpiresAt); + const later = structuredClone(c.input); later.now = now + 400 * 86_400_000; + later.environment = { ...later.environment }; + assert.equal(hostedPolicy(later.defaults, { cellCount: 2, now: later.now, controllerLockSha256: 'e'.repeat(64) }).pricingVerifiedAt, + new Date(later.now).toISOString()); + const capped = structuredClone(c.input); capped.maxUsd = '5'; + assert.equal(prepareHostedManifest(capped).manifest.maxUsd, '5'); +}); + +test('the committed defaults admit the full 41-case Claude plan within their spend cap', async () => { + // Under the portable runtime this reads the packaged copy that hosted.mjs prepare uses. + const defaults = JSON.parse(await readFile(new URL('./hosted-defaults.json', runtime), 'utf8')); + const now = Date.now(), projectId = '4dcaad06-b1ce-4416-ac28-482f1443d5ea'; + const manifestFor = cells => { + const policy = hostedPolicy(defaults, { cellCount: cells, now, projectId, controllerLockSha256: 'e'.repeat(64) }); + const attempts = Array.from({ length: cells }, (_, index) => ({ ...policy.allocations[0], + executionId: `protocol-live-testops-claude--case-${index}`, attemptId: `attempt-${index}`, requestSha256: 'a'.repeat(64) })); + return { schema: 'paperclip.runner-eval.controller/v1', campaignId: 'gha-1-1', maxUsd: policy.maxUsd, + deadlineAt: new Date(now + policy.durationMs).toISOString(), pricingVerifiedAt: policy.pricingVerifiedAt, pricingExpiresAt: policy.pricingExpiresAt, + route: policy.route, projectId: policy.projectId, models: policy.models, overhead: policy.overhead, observability: policy.observability, + sourceRevision: 'b'.repeat(40), definitionsRevision: 'c'.repeat(40), + runnerCli: { path: '/tmp/r/cli.js', sha256: '1'.repeat(64) }, runnerPackage: { path: '/tmp/r/p.tgz', sha256: '2'.repeat(64) }, + runnerd: { path: '/tmp/r/runnerd', sha256: '3'.repeat(64) }, outputDirectory: '/tmp/r/evidence', attempts }; + }; + const full = validateManifest(manifestFor(41), now); + assert.equal(full.manifest.attempts.length, 41); + assert.equal(full.allocatedUsd, "339.000000000"); assert.ok(units(full.allocatedUsd) <= units(defaults.maxUsd)); + assert.throws(() => validateManifest(manifestFor(49), now), /exceed the campaign budget/); }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index c85cd3bd294..995817ed4ba 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -60,7 +60,10 @@ export function validateManifest(value, now = Date.now(), { admission = true } = if (model.price.longContext) { fields(model.price.longContext, ['aboveInputTokens', 'rates']); fields(model.price.longContext.rates, rateFields); } const url = new URL(model.price.sourceUrl); const key = `${model.provider}:${model.price.model}`; - if (!['openai', 'anthropic'].includes(model.provider) || !model.provisionalBound || url.protocol !== 'https:' + // Model-request readiness accounts every request from its receipt; a self-hashed + // full-lifecycle bound is required only when unresolved requests may settle against it. + const boundRequired = value.overhead.accountingReadiness !== 'model_requests'; + if (!['openai', 'anthropic'].includes(model.provider) || boundRequired && !model.provisionalBound || url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || prices.has(key)) throw new Error('Invalid controller model pricing'); prices.set(key, units(costRuntime.evalRequestReservation({ ...model, mode: value.route, apiKey: 'validation' }))); } diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index 9e1ef440986..449ce7fd405 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -92,7 +92,7 @@ test('copied controller runs without source dependencies, retains native evidenc const hosted = await exec(process.execPath, ['--test', '--test-reporter=tap', join(source, 'hosted.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 30000, maxBuffer: 1_048_576, }); - assert.match(hosted.stdout, /# pass 7\b/); + assert.match(hosted.stdout, /# pass 8\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index dae4647342f..7c7425e13e6 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -300,17 +300,21 @@ function workflowShell(workflow, name) { return script.split('\n').map(line => line.slice(10)).join('\n'); } -test('controlled admission rejects missing policy, automatic retries and job reruns before paid work', async () => { +test('controlled admission rejects automatic retries and job reruns without a hand-maintained policy', async () => { const workflow = await readFile(workflowPath, 'utf8'); const script = workflowShell(workflow, 'Validate accounting mode and retry envelope'); const root = await mkdtemp(resolve(tmpdir(), 'runner-workflow-admission-')); try { + // The retired policy variable is ignored whether it is absent or still set. for (const [mode, event, attempt, retries, policy, status] of [ - ['controlled', 'workflow_dispatch', '1', '0', '', 1], + ['controlled', 'workflow_dispatch', '1', '0', '', 0], ['controlled', 'workflow_dispatch', '2', '0', JSON.stringify({schema:'paperclip.runner-eval.hosted-policy/v1',controllerLockSha256:'e'.repeat(64)}), 1], - ['controlled', 'workflow_dispatch', '1', '1', JSON.stringify({schema:'paperclip.runner-eval.hosted-policy/v1',controllerLockSha256:'e'.repeat(64)}), 1], - ['legacy', 'schedule', '1', '1', '', 1], + ['controlled', 'workflow_dispatch', '1', '1', '', 1], + // Schedules are forced to controlled mode with zero retries, so requested legacy retries cannot apply. + ['legacy', 'schedule', '1', '1', '', 0], + ['controlled', 'schedule', '2', '0', '', 1], ['controlled', 'workflow_dispatch', '1', '0', JSON.stringify({schema:'paperclip.runner-eval.hosted-policy/v1',controllerLockSha256:'e'.repeat(64)}), 0], + ['controlled', 'workflow_dispatch', '1', '0', 'not json', 0], ['legacy', 'workflow_dispatch', '1', '1', '', 0], ]) { const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { PATH: process.env.PATH, @@ -375,7 +379,7 @@ test('TestOps admission preserves the selected mode gates, current source and de for (const [mode, source, roster, policy, retries, expected] of [ ['legacy', 'a', 'protocol-live-testops-claude', true, '0', 0], ['controlled', 'a', 'protocol-live-testops-claude', true, '0', 0], - ['controlled', 'a', 'protocol-live-testops-claude', false, '0', 1], + ['controlled', 'a', 'protocol-live-testops-claude', false, '0', 0], ['controlled', 'a', 'protocol-live-testops-claude', true, '1', 1], ['controlled', 'b', 'protocol-live-testops-claude', true, '0', 1], ['legacy', 'b', 'protocol-live-testops-claude', true, '0', 1], @@ -428,14 +432,21 @@ test('controlled capacity counts the filtered selection before any provider work const upload = workflow.indexOf(' - name: Upload immutable campaign catalog'); assert.ok(filter > 0 && filter < limit && limit < upload); const script = workflowShell(workflow, 'Enforce controlled limits against the selected catalog'); - const cell = { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime' }; + const cell = { provider: 'acpx', acpxAgent: 'claude', driver: 'acpx_runtime', model: 'claude-sonnet-5' }; + const codex = { provider: 'codex', driver: 'codex_app_server', model: 'gpt-6' }; const root = await mkdtemp(resolve(tmpdir(), 'runner-selected-capacity-')); try { - for (const [cells, maximum, expected] of [[[cell], 1, 0], [Array(41).fill(cell), 1, 1], - [[cell, cell], 1, 1], [[{ ...cell, provider: 'unsupported' }], 1, 1], [[cell], 0, 1]]) { + const defaults = resolve(root, 'packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json'); + await mkdir(resolve(defaults, '..'), { recursive: true }); + await writeFile(defaults, await readFile(resolve(repositoryRoot, 'packages/paperclip-runner/scripts/controlled-evals/hosted-defaults.json'))); + // Campaign size is the selection itself; the spend cap is enforced during manifest preparation. + for (const [cells, expected] of [[[cell], 0], [Array(41).fill(cell), 0], [Array(512).fill(cell), 0], + [Array(513).fill(cell), 1], [[{ ...cell, provider: 'unsupported' }], 1], [[], 1], + // A supported driver without a committed allocation fails here, not after the builds. + [[cell, codex], 1], [[{ ...cell, model: 'claude-other' }], 1]]) { await writeFile(resolve(root, 'runner-protocol-eval-catalog.json'), JSON.stringify({ cells })); const result = spawnSync('bash', ['-euc', script], { cwd: root, encoding: 'utf8', env: { PATH: process.env.PATH, - CONTROLLER_POLICY_JSON: JSON.stringify({ maxCells: maximum }) } }); + CONTROLLER_POLICY_JSON: JSON.stringify({ maxCells: 1 }) } }); assert.equal(result.status === 0, expected === 0, result.stderr); } } finally { await rm(root, { recursive: true, force: true }); } @@ -483,7 +494,13 @@ test('private archive jobs consume verified controller code without resolving de assert.doesNotMatch(builder, /id-token: write|environment:|secrets\./u); assert.match(builder, /build-runtime.mjs/u); assert.doesNotMatch(builder, /--no-frozen-lockfile/u); - assert.match(builder, /test "\$RESOLVED_LOCK_SHA256" = "\$EXPECTED_LOCK_SHA256"/u); + assert.doesNotMatch(builder, /target_lock|RESOLVED_LOCK_SHA256|runner-controller-lock/u); + assert.match(builder, /git diff --quiet HEAD -- pnpm-lock\.yaml/u); + assert.match(preparer, /committed="\$\(sha256sum pnpm-lock\.yaml/u); + assert.match(preparer, /PAPERCLIP_RUNNER_EVAL_CONTROLLER_LOCK_SHA256: \$\{\{ steps\.controller_lock\.outputs\.sha256 \}\}/u); + // Defaults come from the verified controller runtime, not the workspace that evals code can modify. + assert.doesNotMatch(preparer, /hosted-defaults\.json/u); + assert.doesNotMatch(workflow, /RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON/u); assert.match(builder, /pnpm install --frozen-lockfile --ignore-scripts --store-dir/u); assert.doesNotMatch(preparer, /pnpm install|build-runtime.mjs|cache: pnpm/u); assert.ok(preparer.indexOf('verify-runtime.mjs') < preparer.indexOf('hosted.mjs" prepare')); @@ -498,23 +515,20 @@ test('private archive jobs consume verified controller code without resolving de } }); -test('controller dependency lock rejects resolution or byte drift before installation', async () => { +test('controller dependency lock rejects a modified committed lockfile before installation', async () => { const workflow = await readFile(workflowPath, 'utf8'); const script = workflowShell(workflow, 'Build portable controller in an unprivileged job'); - const pin = script.slice(script.indexOf('test "$RESOLVED_LOCK_SHA256"'), script.indexOf('pnpm install')); - const root = await mkdtemp(resolve(tmpdir(), 'runner-reviewed-lock-')); - const digest = createHash('sha256').update('reviewed dependencies').digest('hex'); + const guard = script.slice(0, script.indexOf('pnpm install')); + const root = await mkdtemp(resolve(tmpdir(), 'runner-committed-lock-')); + const git = (...args) => spawnSync('git', args, { cwd: root, encoding: 'utf8' }); try { - await mkdir(resolve(root, 'runner-controller-lock')); - for (const [index, resolved, bytes, status] of [[0, digest, 'reviewed dependencies', 0], [1, 'f'.repeat(64), 'reviewed dependencies', 1], [2, digest, 'changed dependencies', 1]]) { - const cwd = resolve(root, `worker-${index}`); await mkdir(cwd); - await writeFile(resolve(root, 'runner-controller-lock/pnpm-lock.yaml'), bytes); - const result = spawnSync('bash', ['-euc', pin], { cwd, encoding: 'utf8', env: { - PATH: process.env.PATH, RUNNER_TEMP: root, EXPECTED_LOCK_SHA256: digest, RESOLVED_LOCK_SHA256: resolved, - } }); - assert.equal(result.status, status, result.stderr); - if (status === 0) assert.equal(await readFile(resolve(cwd, 'pnpm-lock.yaml'), 'utf8'), bytes); - else await assert.rejects(readFile(resolve(cwd, 'pnpm-lock.yaml')), { code: 'ENOENT' }); - } + git('init', '-q'); git('config', 'user.email', 'test@example.com'); git('config', 'user.name', 'test'); + await writeFile(resolve(root, 'pnpm-lock.yaml'), 'committed dependencies'); + git('add', 'pnpm-lock.yaml'); git('commit', '-qm', 'lock'); + const head = git('rev-parse', 'HEAD').stdout.trim(); + const run = () => spawnSync('bash', ['-euc', guard], { cwd: root, encoding: 'utf8', env: { PATH: process.env.PATH, GITHUB_SHA: head } }); + assert.equal(run().status, 0, run().stderr); + await writeFile(resolve(root, 'pnpm-lock.yaml'), 'changed dependencies'); + assert.notEqual(run().status, 0); } finally { await rm(root, { recursive: true, force: true }); } });