diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index cf58de85b3d..eca9ea71bfe 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -381,16 +381,18 @@ jobs: GROK_AUTHENTICATION: ${{ inputs.grok_authentication || 'api_key' }} run: | set -euo pipefail - if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then - echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 2 through $MAX_PARALLEL_LIMIT for the two-shard direct suite." >&2 + if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ]; then + echo "RUNNER_E2E_MAX_PARALLEL must be an integer of at least 2 for the two-shard direct suite." >&2 exit 1 fi if [ -n "${REQUESTED_MAX_PARALLEL:-}" ]; then - if ! [[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$REQUESTED_MAX_PARALLEL" -lt 2 ] || [ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]; then - echo "max_parallel must be an integer from 2 through the configured campaign limit." >&2 + if ! [[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$REQUESTED_MAX_PARALLEL" -lt 2 ] || [ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ] || [ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then + echo "max_parallel must be an integer from 2 through both configured campaign and runner limits." >&2 exit 1 fi MAX_PARALLEL="$REQUESTED_MAX_PARALLEL" + elif [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then + MAX_PARALLEL="$MAX_PARALLEL_LIMIT" fi node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \ --evals-root .paperclip-evals \ @@ -850,6 +852,15 @@ jobs: artifact-ids: ${{ needs.prepare_controller.outputs.artifact_id }} path: /tmp/paperclip-runner-controlled-${{ github.run_id }}/runtime + - name: Stage an owned provider Node before credentials + run: | + set -euo pipefail + mkdir -p cell-output + node packages/paperclip-runner/scripts/stage-runner-provider-node.mjs \ + "$RUNNER_TEMP/runner-protocol-provider-node" \ + "$GITHUB_WORKSPACE/runner-protocol-build/extracted/portable/dist/cli/acpx-runtime-sidecar.cjs" \ + cell-output/provider-node-receipt.json + - name: Verify controller code before archive access if: needs.authorize.outputs.accounting_mode == 'controlled' id: verify_controller diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 5daed9da60a..dece4dc0e67 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -822,6 +822,15 @@ entry file. Installed dependencies still belong to that trusted checkout. The staged copies are removed after verified process cleanup and kept if cleanup is incomplete. This path requires the repository's Node 24.11+ runtime. +Each Linux protocol worker snapshots its current Node executable into a fresh +owner-only directory before credential steps. The copy keeps the same bytes +and version, with directory mode `0700` and file mode `0500`. The worker verifies +the copy's hash and actual `process.execPath`, then puts that directory first +on its job-local `PATH`. `cell-output/provider-node-receipt.json` records the +original and copied Node modes and hashes plus the actual sidecar mode and +hash. This handles writable hosted tool caches without changing those caches +or relaxing Runner's qualified-artifact permission and digest checks. + Campaign reports verify the retained request binding and the digest of every request receipt before reducing a cell's allocation. Missing admission, request, or receipt evidence preserves known ledger charges and holds the rest of the @@ -990,6 +999,8 @@ Controlled mode preserves its reviewed policy, immutable lock/source/request bindings, tracing, private retention, first-attempt and zero-retry gates. Its paid workers use `ubuntu-latest`, matching the required GitHub-hosted operating origin even when the legacy AWS fleet selector is enabled. +Requested parallelism must fit both the configured campaign ceiling and the +selected runner limit. Without a request, the lower ceiling applies. Legacy mode uses direct-provider credentials. A pass in one mode does not prove the other's credential or routing path. @@ -998,6 +1009,10 @@ job run, launch, standing plan and exact selected case identities. The full 41-case roster is filtered to that selection before matrix fan-out. A manual reporting run validates the complete standing plan and executes only existing case 194 (`get-task-context`) to register the GitHub job with real evidence. +A manual run captures numeric job-run and launch identities from the supported +`allurectl job-run env` command inside the watch session; watch does not fill +an empty `ALLURE_JOB_RUN_ID` input. The captured launch must match the +authoritative job-run response. Service-triggered runs keep their supplied ID. A TestOps-triggered job uses its requested selection. Targeted selections retain their smaller coverage and omit the `full-run` launch tag. Full selections use `full-run` and `test-plan-3`. diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 7f12eb5ce52..dff8204861f 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/materialize-pi-distribution.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/acpx-release-attestations.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-kimi-distribution.test.mjs scripts/provision-kimi-package.test.mjs scripts/kimi-metadata-probe.test.mjs scripts/patch-kimi-runtime.test.mjs test/kimi-installed-runtime.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/contracts/contract-fingerprints.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/materialize-pi-distribution.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/acpx-release-attestations.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-kimi-distribution.test.mjs scripts/provision-kimi-package.test.mjs scripts/kimi-metadata-probe.test.mjs scripts/patch-kimi-runtime.test.mjs test/kimi-installed-runtime.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-provider-node.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/contracts/contract-fingerprints.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", @@ -132,7 +132,7 @@ "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", - "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-testops.test.mjs scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-metrics.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs scripts/public-eval-chat.test.mjs", + "test:runner-protocol-eval-publish": "node --test scripts/stage-runner-provider-node.test.mjs scripts/runner-protocol-testops.test.mjs scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-metrics.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs scripts/public-eval-chat.test.mjs", "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", "report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs", diff --git a/packages/paperclip-runner/scripts/provision-kimi-package.test.mjs b/packages/paperclip-runner/scripts/provision-kimi-package.test.mjs index 1afbc0627b5..acd9f190df9 100644 --- a/packages/paperclip-runner/scripts/provision-kimi-package.test.mjs +++ b/packages/paperclip-runner/scripts/provision-kimi-package.test.mjs @@ -55,8 +55,9 @@ test("installed Kimi setup is self-contained, rejects corrupt assets and cancels const kill = setTimeout(() => cancelled.kill("SIGKILL"), 5_000); try { for (let attempt = 0; attempt < 100; attempt++) { - try { await readFile(ready); break; } - catch (error) { if (error.code !== "ENOENT") throw error; await delay(10); } + try { if (await readFile(ready, "utf8") === "ready") break; } + catch (error) { if (error.code !== "ENOENT") throw error; } + await delay(10); } assert.equal(await readFile(ready, "utf8"), "ready"); cancelled.kill("SIGTERM"); diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index 08a81d80727..b15dc628f36 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -337,6 +337,36 @@ test('controlled workers keep the required GitHub-hosted origin even when the le } finally { await rm(root, { recursive: true, force: true }); } }); +test('requested parallelism respects both configured capacity and the selected runner cap', async () => { + const workflow = await readFile(workflowPath, 'utf8'); + const script = workflowShell(workflow, 'Build the two bounded roster-plus-case matrices'); + const prefix = script.slice(0, script.indexOf('node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs')); + assert.ok(prefix.includes('REQUESTED_MAX_PARALLEL')); + for (const [configured, limit, requested, expected] of [['100', '57', '2', '2'], ['100', '57', '', '57'], + ['100', '100', '', '100'], ['8', '57', '16', null], ['100', '57', '58', null], ['100', '57', '1', null]]) { + const result = spawnSync('bash', ['-euc', `${prefix}\nprintf '%s' "$MAX_PARALLEL"`], { encoding: 'utf8', env: { + PATH: process.env.PATH, MAX_PARALLEL: configured, MAX_PARALLEL_LIMIT: limit, REQUESTED_MAX_PARALLEL: requested } }); + assert.equal(result.status === 0, expected !== null, `${configured}/${limit}/${requested}: ${result.stderr}`); + if (expected !== null) assert.equal(result.stdout, expected); + } +}); + +test('the provider uses an owned same-byte Node before credentials without changing shared host permissions', async () => { + const workflow = await readFile(workflowPath, 'utf8'); + const paid = workflow.slice(workflow.indexOf(' steps: &direct_eval_steps'), workflow.indexOf(' eval_shard_1:')); + const staging = paid.indexOf('Stage an owned provider Node before credentials'); + assert.ok(staging > paid.indexOf('Verify and extract portable runner')); + assert.ok(staging < paid.indexOf('Obtain private archive credentials for transfer steps only')); + assert.ok(staging < paid.indexOf('Prepare short-lived AgentCore web identity')); + assert.ok(staging < paid.indexOf('Run one immutable direct protocol cell')); + const command = workflowShell(paid, 'Stage an owned provider Node before credentials'); + assert.match(command, /node packages\/paperclip-runner\/scripts\/stage-runner-provider-node\.mjs/u); + assert.match(command, /"\$RUNNER_TEMP\/runner-protocol-provider-node"/u); + assert.match(command, /portable\/dist\/cli\/acpx-runtime-sidecar\.cjs/u); + assert.match(command, /cell-output\/provider-node-receipt\.json/u); + assert.doesNotMatch(command, /secrets\.|\bsudo\b|\bchmod\b|hostedtoolcache/u); +}); + test('TestOps admission preserves the selected mode gates, current source and dedicated roster', async () => { const workflow = await readFile(workflowPath, 'utf8'); const script = workflowShell(workflow, 'Validate accounting mode and retry envelope'); diff --git a/packages/paperclip-runner/scripts/runner-protocol-testops.mjs b/packages/paperclip-runner/scripts/runner-protocol-testops.mjs index 78ab2d3d038..79d37aab983 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-testops.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-testops.mjs @@ -1,6 +1,8 @@ #!/usr/bin/env node import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; import { createHash, randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { failureProjection, testopsClient } from "../../../tests/runner-e2e/chat-feature-testops-api.mjs"; @@ -12,6 +14,35 @@ const sha = value => typeof value === "string" && /^[a-f0-9]{40}$/.test(value); const json = value => `${JSON.stringify(value, null, 2)}\n`; const workflowUrl = "https://github.com/paperclipai/paperclip/actions/workflows/runner-protocol-live-evals.yml"; const runUrl = runId => `https://github.com/paperclipai/paperclip/actions/runs/${runId}`; +const officialExec = promisify(execFile); + +export function parseOfficialJobRunContext(stdout) { + if (typeof stdout !== "string" || Buffer.byteLength(stdout) > 16_384) fail("bootstrap_context"); + const values = new Map(); + for (const line of stdout.split(/\r?\n/)) { + for (const name of ["ALLURE_JOB_RUN_ID", "ALLURE_LAUNCH_ID"]) { + if (!line.startsWith(`${name}=`)) continue; + const value = line.slice(name.length + 1); + if (values.has(name) || !/^[1-9][0-9]{0,15}$/.test(value) || !positive(Number(value))) fail("bootstrap_context"); + values.set(name, value); + } + } + if (values.size !== 2) fail("bootstrap_context"); + return { jobRunId: values.get("ALLURE_JOB_RUN_ID"), launchId: Number(values.get("ALLURE_LAUNCH_ID")) }; +} + +export async function captureManualJobRunContext(env, exec = officialExec) { + if (env.RUNNER_TESTOPS_REQUESTED_JOB_RUN_ID !== "" || env.ALLURE_ENDPOINT !== "https://paperclip.testops.cloud" + || env.ALLURE_PROJECT_ID !== String(contract.projectId)) fail("bootstrap_context"); + let stdout; + try { + ({ stdout } = await exec("allurectl", ["job-run", "env"], { env, shell: false, + encoding: "utf8", timeout: 30_000, maxBuffer: 16_384, windowsHide: true })); + } catch { fail("bootstrap_context"); } + const captured = parseOfficialJobRunContext(stdout); + if (env.ALLURE_JOB_RUN_ID && env.ALLURE_JOB_RUN_ID !== captured.jobRunId) fail("bootstrap_context"); + return { ...env, ALLURE_JOB_RUN_ID: captured.jobRunId, RUNNER_TESTOPS_EXPECTED_LAUNCH_ID: String(captured.launchId) }; +} export function standingPlanCases(plan) { if (plan?.version !== "1.0" || !Array.isArray(plan.tests)) fail("standing_plan"); @@ -56,6 +87,7 @@ export async function inspectSelection(api, env, manualPlan) { const acceptedUrls = [workflowUrl, `https://github.com/paperclipai/paperclip/actions/workflows/${contract.workflowId}`, "https://github.com/paperclipai/paperclip/blob/master/.github/workflows/runner-protocol-live-evals.yml"]; if (run.id !== jobRunId || !positive(run.job?.id) || run.job.type !== undefined && run.job.type !== "github" || run.job.url !== undefined && !acceptedUrls.includes(run.job.url) || run.url !== runUrl(env.GITHUB_RUN_ID) + || manualPlan && Number(env.RUNNER_TESTOPS_EXPECTED_LAUNCH_ID) !== run.launchId || !["scheduled", "in_progress"].includes(run.stage)) fail("jobrun_binding"); const [job, launch, plan, planned] = await Promise.all([ api(`/api/job/${run.job.id}`), @@ -200,8 +232,9 @@ export function protocolTestOpsFailure(error) { async function main() { const [command, ...args] = process.argv.slice(2); if (command === "selection") { - const selection = await inspectSelection(await testopsClient(process.env.ALLURE_TOKEN), process.env, - args[1] === undefined ? undefined : await readJson(args[1])); + const manualPlan = args[1] === undefined ? undefined : await readJson(args[1]); + const env = manualPlan ? await captureManualJobRunContext(process.env) : process.env; + const selection = await inspectSelection(await testopsClient(env.ALLURE_TOKEN), env, manualPlan); await writeFile(args[0], json(selection), { mode: 0o600 }); await githubOutputs({ allure_job_run_id: selection.allureJobRunId, full_plan: String(selection.fullPlan) }); console.log(json({ selected: selection.cases.length, fullPlan: selection.fullPlan, launchId: selection.launchId })); diff --git a/packages/paperclip-runner/scripts/runner-protocol-testops.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-testops.test.mjs index cff6829e3fe..cfa240e67b8 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-testops.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-testops.test.mjs @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { contract, selectCases, inspectSelection, filterCatalog, projectResults, verifyPublication, standingPlanCases, protocolTestOpsFailure } from "./runner-protocol-testops.mjs"; +import { contract, selectCases, inspectSelection, filterCatalog, projectResults, verifyPublication, standingPlanCases, protocolTestOpsFailure, + captureManualJobRunContext, parseOfficialJobRunContext } from "./runner-protocol-testops.mjs"; import { testopsClient } from "../../../tests/runner-e2e/chat-feature-testops-api.mjs"; const context = { runId: "123", workflowSha: "a".repeat(40), sourceSha: "a".repeat(40), evalsSha: "b".repeat(40), @@ -66,10 +67,14 @@ test("binds the selected TestOps job to this exact workflow run and standing pla delete run.job.url; assert.equal((await inspectSelection(api, env)).cases.length, 1); const manual = { version: "1.0", tests: planned.map(row => ({ ...row, id: String(row.id) })) }; - const admittedManual = await inspectSelection(api, { ...env, RUNNER_TESTOPS_REQUESTED_JOB_RUN_ID: "" }, manual); + const manualEnv = await captureManualJobRunContext({ ...env, ALLURE_JOB_RUN_ID: "", RUNNER_TESTOPS_REQUESTED_JOB_RUN_ID: "", + ALLURE_ENDPOINT: "https://paperclip.testops.cloud", ALLURE_PROJECT_ID: "1" }, + async () => ({ stdout: "ALLURE_JOB_RUN_ID=7\nALLURE_LAUNCH_ID=11\n" })); + const admittedManual = await inspectSelection(api, manualEnv, manual); assert.equal(admittedManual.fullPlan, false); assert.equal(admittedManual.cases.length, 1); assert.equal(admittedManual.cases[0].testCaseId, 194); + await assert.rejects(inspectSelection(api, { ...manualEnv, RUNNER_TESTOPS_EXPECTED_LAUNCH_ID: "12" }, manual), /jobrun_binding/); const controlled = await inspectSelection(api, { ...env, PAPERCLIP_PROTOCOL_ACCOUNTING_MODE: "controlled" }); assert.equal(controlled.accountingMode, "controlled"); assert.equal(projectResults({ selection: controlled })[0].evidence.accountingMode, "controlled"); @@ -82,6 +87,32 @@ test("binds the selected TestOps job to this exact workflow run and standing pla await assert.rejects(inspectSelection(api, env), /jobrun_binding/); }); +test("captures only official numeric manual job context without evaluating or exposing its environment", async () => { + const env = { RUNNER_TESTOPS_REQUESTED_JOB_RUN_ID: "", ALLURE_JOB_RUN_ID: "", + ALLURE_ENDPOINT: "https://paperclip.testops.cloud", ALLURE_PROJECT_ID: "1" }; + const captured = await captureManualJobRunContext(env, async (command, args, options) => { + assert.equal(command, "allurectl"); + assert.deepEqual(args, ["job-run", "env"]); + assert.equal(options.shell, false); + assert.equal(options.timeout, 30_000); + assert.equal(options.maxBuffer, 16_384); + return { stdout: "ALLURE_TOKEN=PRIVATE-SECRET\nALLURE_JOB_RUN_ID=2550\r\nALLURE_LAUNCH_ID=2588\n" }; + }); + assert.equal(captured.ALLURE_JOB_RUN_ID, "2550"); + assert.equal(captured.RUNNER_TESTOPS_EXPECTED_LAUNCH_ID, "2588"); + assert.ok(!JSON.stringify(captured).includes("PRIVATE-SECRET")); + for (const text of ["", "ALLURE_JOB_RUN_ID=7\n", "ALLURE_JOB_RUN_ID=7\nALLURE_LAUNCH_ID=0\n", + "ALLURE_JOB_RUN_ID=7\nALLURE_LAUNCH_ID=11\nALLURE_JOB_RUN_ID=7\n", + "ALLURE_JOB_RUN_ID=$(private)\nALLURE_LAUNCH_ID=11\n", "x".repeat(16_385)]) { + assert.throws(() => parseOfficialJobRunContext(text), /bootstrap_context/); + } + await assert.rejects(captureManualJobRunContext({ ...env, ALLURE_JOB_RUN_ID: "9" }, + async () => ({ stdout: "ALLURE_JOB_RUN_ID=7\nALLURE_LAUNCH_ID=11\n" })), /bootstrap_context/); + await assert.rejects(captureManualJobRunContext({ ...env, RUNNER_TESTOPS_REQUESTED_JOB_RUN_ID: "7" }, + async () => { throw new Error("must not execute"); }), /bootstrap_context/); + await assert.rejects(captureManualJobRunContext(env, async () => { throw new Error("PRIVATE-SECRET"); }), /bootstrap_context/); +}); + test("reports every unexecuted selected case as broken when the report is unavailable", () => { const rows = projectResults({ selection: selection(planned) }); assert.equal(rows.length, 41); diff --git a/packages/paperclip-runner/scripts/stage-runner-provider-node.mjs b/packages/paperclip-runner/scripts/stage-runner-provider-node.mjs new file mode 100644 index 00000000000..53643c6cdce --- /dev/null +++ b/packages/paperclip-runner/scripts/stage-runner-provider-node.mjs @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { appendFileSync, chmodSync, closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, rmSync, writeFileSync } from 'node:fs'; +import { isAbsolute, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const mode = metadata => metadata.mode & 0o777; +const stable = (left, right) => ['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'].every(key => left[key] === right[key]); + +function readArtifact(file, maximumBytes) { + const before = lstatSync(file); + if (!before.isFile() || before.isSymbolicLink() || before.size < 1 || before.size > maximumBytes) throw new Error('Provider artifact must be a bounded regular file'); + const descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const opened = fstatSync(descriptor); + if (!opened.isFile() || !stable(before, opened)) throw new Error('Provider artifact changed before reading'); + const bytes = Buffer.alloc(opened.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error('Provider artifact changed while reading'); + offset += count; + } + if (readSync(descriptor, Buffer.alloc(1), 0, 1, offset) || !stable(opened, fstatSync(descriptor))) throw new Error('Provider artifact changed while reading'); + return { bytes, metadata: opened, path: file, mode: mode(opened), sha256: hash(bytes) }; + } finally { closeSync(descriptor); } +} + +// The workflow trusts setup-node's current executable. Snapshot those same +// bytes into an owned location; never change the shared hosted tool cache. +export function stageExecutableSnapshot(source, directory) { + if (!isAbsolute(directory) || /[\r\n]/u.test(directory)) throw new Error('Provider Node needs an absolute owned directory'); + const original = readArtifact(source, 256 * 1024 * 1024); + if (!(original.mode & 0o111)) throw new Error('Provider Node source is not executable'); + mkdirSync(directory, { mode: 0o700 }); + try { + chmodSync(directory, 0o700); + const destination = join(directory, 'node'); + writeFileSync(destination, original.bytes, { flag: 'wx', mode: 0o500 }); + chmodSync(destination, 0o500); + const copied = readArtifact(destination, 256 * 1024 * 1024); + const parent = lstatSync(directory); + if (copied.sha256 !== original.sha256 || copied.mode !== 0o500 || !parent.isDirectory() || mode(parent) !== 0o700 + || copied.metadata.uid !== process.geteuid() || parent.uid !== process.geteuid()) throw new Error('Provider Node snapshot is not private and byte-identical'); + return { source: { path: source, mode: original.mode, sha256: original.sha256 }, + staged: { path: destination, mode: copied.mode, sha256: copied.sha256 }, directoryMode: mode(parent) }; + } catch (error) { rmSync(directory, { recursive: true, force: true }); throw error; } +} + +export function stageProviderNode({ directory, sidecar }) { + if (process.platform !== 'linux') throw new Error('Protocol provider Node staging requires Linux'); + const sidecarArtifact = readArtifact(sidecar, 16 * 1024 * 1024); + if (sidecarArtifact.mode & 0o022) throw new Error('Provider sidecar must not be group- or world-writable'); + const snapshot = stageExecutableSnapshot(process.execPath, directory); + try { + const probe = JSON.parse(execFileSync(snapshot.staged.path, ['--input-type=module', '-e', + 'import {readFileSync,lstatSync} from "node:fs";import {createHash} from "node:crypto";process.stdout.write(JSON.stringify({execPath:process.execPath,version:process.version,mode:lstatSync(process.execPath).mode&511,sha256:createHash("sha256").update(readFileSync(process.execPath)).digest("hex")}));'], + { env: { PATH: process.env.PATH ?? '/usr/bin:/bin' }, encoding: 'utf8', timeout: 10_000, maxBuffer: 4096, stdio: ['ignore', 'pipe', 'pipe'] })); + if (probe.execPath !== snapshot.staged.path || probe.version !== process.version || probe.mode !== 0o500 || probe.sha256 !== snapshot.source.sha256) throw new Error('Provider Node did not execute the admitted bytes'); + return { schema: 'paperclip.runner-provider-node/v1', platform: process.platform, arch: process.arch, version: probe.version, + ...snapshot, sidecar: { path: sidecar, mode: sidecarArtifact.mode, sha256: sidecarArtifact.sha256 } }; + } catch (error) { rmSync(directory, { recursive: true, force: true }); throw error; } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + if (process.argv.length !== 5 || !process.env.GITHUB_PATH) throw new Error('Provider Node staging needs directory, sidecar, receipt and GitHub path context'); + const receipt = stageProviderNode({ directory: process.argv[2], sidecar: process.argv[3] }); + writeFileSync(process.argv[4], `${JSON.stringify(receipt, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); + appendFileSync(process.env.GITHUB_PATH, `${process.argv[2]}\n`); + console.log(JSON.stringify(receipt)); + } catch { console.error('Provider Node staging failed before credentials or provider execution.'); process.exitCode = 1; } +} diff --git a/packages/paperclip-runner/scripts/stage-runner-provider-node.test.mjs b/packages/paperclip-runner/scripts/stage-runner-provider-node.test.mjs new file mode 100644 index 00000000000..ad9dba69ca8 --- /dev/null +++ b/packages/paperclip-runner/scripts/stage-runner-provider-node.test.mjs @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { chmodSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; +import { stageExecutableSnapshot, stageProviderNode } from './stage-runner-provider-node.mjs'; + +test('a writable executable becomes a byte-identical private snapshot without changing its source', t => { + const root = mkdtempSync(join(tmpdir(), 'provider-node-snapshot-')); t.after(() => rmSync(root, { recursive: true, force: true })); + const source = join(root, 'host-node'); writeFileSync(source, 'same executable bytes'); chmodSync(source, 0o777); + const receipt = stageExecutableSnapshot(source, join(root, 'private')); + assert.deepEqual(readFileSync(receipt.staged.path), readFileSync(source)); + assert.equal(receipt.source.mode, 0o777); assert.equal(lstatSync(source).mode & 0o777, 0o777); + assert.equal(receipt.staged.mode, 0o500); assert.equal(receipt.directoryMode, 0o700); + assert.equal(receipt.source.sha256, receipt.staged.sha256); +}); + +test('links, directories, nonexecutables and reused output directories cannot become provider Node', t => { + const root = mkdtempSync(join(tmpdir(), 'provider-node-rejections-')); t.after(() => rmSync(root, { recursive: true, force: true })); + const source = join(root, 'host-node'); writeFileSync(source, 'bytes'); chmodSync(source, 0o755); + const linked = join(root, 'linked'); symlinkSync(source, linked); + assert.throws(() => stageExecutableSnapshot(linked, join(root, 'link-output')), /regular file/u); + assert.throws(() => stageExecutableSnapshot(root, join(root, 'directory-output')), /regular file/u); + chmodSync(source, 0o644); + assert.throws(() => stageExecutableSnapshot(source, join(root, 'not-executable')), /not executable/u); + chmodSync(source, 0o755); const existing = join(root, 'existing'); mkdirSync(existing); writeFileSync(join(existing, 'sentinel'), 'preserve'); + assert.throws(() => stageExecutableSnapshot(source, existing), /EEXIST/u); + assert.equal(readFileSync(join(existing, 'sentinel'), 'utf8'), 'preserve'); + assert.throws(() => stageExecutableSnapshot(source, 'relative'), /absolute/u); + assert.throws(() => stageExecutableSnapshot(source, `${root}\ninjected`), /absolute/u); +}); + +test('the wrapper binds real Node on Linux and rejects other platforms or unsafe sidecars', t => { + const root = mkdtempSync(join(tmpdir(), 'provider-node-linux-')); t.after(() => rmSync(root, { recursive: true, force: true })); + const sidecar = join(root, 'sidecar.cjs'); writeFileSync(sidecar, '// fixture sidecar'); chmodSync(sidecar, 0o644); + if (process.platform !== 'linux') { + assert.throws(() => stageProviderNode({ directory: join(root, 'owned'), sidecar }), /requires Linux/u); + return; + } + const receipt = stageProviderNode({ directory: join(root, 'owned'), sidecar }); + assert.equal(receipt.version, process.version); assert.equal(receipt.staged.path, join(root, 'owned/node')); + assert.equal(receipt.source.sha256, receipt.staged.sha256); assert.equal(receipt.sidecar.mode, 0o644); + assert.equal(execFileSync(receipt.staged.path, ['-p', 'process.execPath'], { encoding: 'utf8' }).trim(), receipt.staged.path); + chmodSync(sidecar, 0o666); + assert.throws(() => stageProviderNode({ directory: join(root, 'unsafe-sidecar'), sidecar }), /sidecar/u); +}); + +test('the workflow CLI records closed modes and hashes and exposes only the private executable directory', t => { + const root = mkdtempSync(join(tmpdir(), 'provider-node-cli-')); t.after(() => rmSync(root, { recursive: true, force: true })); + const sidecar = join(root, 'sidecar.cjs'); writeFileSync(sidecar, '// fixture sidecar'); chmodSync(sidecar, 0o644); + const output = join(root, 'path'), receiptPath = join(root, 'receipt.json'), directory = join(root, 'owned'); + const args = [resolve(import.meta.dirname, 'stage-runner-provider-node.mjs'), directory, sidecar, receiptPath]; + if (process.platform !== 'linux') { + assert.throws(() => execFileSync(process.execPath, args, { env: { PATH: process.env.PATH, GITHUB_PATH: output }, stdio: 'pipe' })); + return; + } + execFileSync(process.execPath, args, { env: { PATH: process.env.PATH, GITHUB_PATH: output, ANTHROPIC_API_KEY: 'must-not-appear' }, stdio: 'pipe' }); + const raw = readFileSync(receiptPath, 'utf8'), receipt = JSON.parse(raw); + assert.equal(receipt.schema, 'paperclip.runner-provider-node/v1'); assert.equal(receipt.staged.mode, 0o500); + assert.equal(readFileSync(output, 'utf8'), `${directory}\n`); assert.doesNotMatch(raw, /must-not-appear|ANTHROPIC_API_KEY/u); +}); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index a507f3034f8..f7beb9ceb02 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -878,7 +878,7 @@ test("direct protocol concurrency override only lowers the configured ceiling", const script = workflow.slice(start, end) + '\nprintf "%s" "$MAX_PARALLEL"\n'; for (const [requested, expected] of [["", "8"], ["2", "2"], ["8", "8"], ["1", null], ["9", null], ["0", null], ["-1", null], ["2.5", null], ["garbage", null], ["9999999999999999999999", null]]) { const result = spawnSync("bash", ["-eu", "-c", script], { - env: { ...process.env, MAX_PARALLEL: "8", REQUESTED_MAX_PARALLEL: requested }, encoding: "utf8", + env: { ...process.env, MAX_PARALLEL: "8", MAX_PARALLEL_LIMIT: "57", REQUESTED_MAX_PARALLEL: requested }, encoding: "utf8", }); assert.equal(result.status, expected === null ? 1 : 0, requested); if (expected !== null) assert.equal(result.stdout, expected);