diff --git a/.github/workflows/chat-catalog-testops.yml b/.github/workflows/chat-catalog-testops.yml index 52360e6af47..30eff3032a5 100644 --- a/.github/workflows/chat-catalog-testops.yml +++ b/.github/workflows/chat-catalog-testops.yml @@ -218,9 +218,9 @@ jobs: build_artifact_id: ${{ needs.build.outputs.artifact_id }} ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} secrets: - OPENROUTER_API_KEY: '' - ANTHROPIC_API_KEY: '' - ALLURE_TOKEN: '' + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} chat_external: name: ${{ matrix.executionId }} @@ -245,9 +245,9 @@ jobs: build_artifact_id: ${{ needs.build.outputs.artifact_id }} ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} secrets: - OPENROUTER_API_KEY: '' - ANTHROPIC_API_KEY: '' - ALLURE_TOKEN: '' + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} aggregate: name: Validate and merge every selected metadata-only projection diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index c4a430c95c4..5167a32193f 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -643,7 +643,8 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { if (name === "chat-catalog-testops.yml") { assert.match(triggerHeader, /^ workflow_dispatch:/m); assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_call|workflow_run):/m); - assert.doesNotMatch(workflow, providerSecretReference); + const callerBlocks = ["chat", "chat_external"].map(id => workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0]); + assert.doesNotMatch(callerBlocks.reduce((rest, block) => rest.replace(block, ""), workflow), providerSecretReference); assert.match(workflow, /group: chat-catalog-testops-streaming-v1/); for (const id of ["chat", "chat_external"]) { const block = workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0]; @@ -653,14 +654,23 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { assert.match(block, /needs: \[authorize, select, build\]/); assert.match(block, /uses: \.\/\.github\/workflows\/chat-catalog-testops-case\.yml/); assert.match(block, /protocol: streaming-v1/); - assert.match(block, /secrets:\n OPENROUTER_API_KEY: ''\n ANTHROPIC_API_KEY: ''\n ALLURE_TOKEN: ''/); - assert.doesNotMatch(block, /secrets: inherit|secrets\./); + const mapping = block.split("\n secrets:\n"); + assert.equal(mapping.length, 2); + assert.deepEqual(mapping[1].trim().split("\n").map(line => line.trim()), [ + "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}", + "ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}", + "ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}", + ]); + assert.doesNotMatch(block, /secrets: inherit/); assert.match(block, /fail-fast: false/); }; assertCaller(block); const lane = id === "chat" ? "native" : "external"; for (const mutant of [block.replace("needs.authorize.outputs.operation == 'evaluate' && ", ""), - block.replace(`needs.select.outputs.${lane}_count != '0'`, 'true')]) assert.throws(() => assertCaller(mutant)); + block.replace(`needs.select.outputs.${lane}_count != '0'`, 'true'), + block.replace('${{ secrets.OPENROUTER_API_KEY }}', "''"), + block + '\n EXTRA: ${{ secrets.EXTRA }}', + block.replace(/\n secrets:[\s\S]*/, '\n secrets: inherit')]) assert.throws(() => assertCaller(mutant)); } const final = workflow.split(" verify:\n")[1]; assert.match(final, /chat-completion-testops-cli\.mjs verify-stream/); diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs index 4d6a6a1f21c..8e8a462c74c 100644 --- a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs +++ b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs @@ -218,7 +218,7 @@ test('the real CLI prepares without TestOps credentials and retains closed faile } }); -test('source host admission follows only the fixed same-commit callee with empty caller secret placeholders', async () => { +test('source host admission follows only the fixed same-commit callee with exact named caller secret references', async () => { const { parse } = await import('yaml'); const { paidWorker } = await import('./chat-completion-testops-host-control.mjs'); const main = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); @@ -227,7 +227,7 @@ test('source host admission follows only the fixed same-commit callee with empty for (const change of [ value => { value.jobs.chat.uses = 'foreign/repository/workflow.yml@master'; }, value => { value.jobs.chat.secrets = 'inherit'; }, - value => { value.jobs.chat_external.secrets.ALLURE_TOKEN = '${{ secrets.ALLURE_TOKEN }}'; }, + value => { value.jobs.chat_external.secrets.ALLURE_TOKEN = ''; }, value => { value.jobs.chat.with.selection_sha256 = 'arbitrary'; }, ]) { const mutant = structuredClone(main); change(mutant); assert.throws(() => paidWorker(mutant, callee), /boundary/); } const wrong = structuredClone(callee); wrong.jobs.chat.environment = 'runner-e2e-testops-reporting'; diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs index 5902a391c89..01bd9517951 100644 --- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -7,6 +7,26 @@ import path from 'node:path'; import { parse } from 'yaml'; import { admitDispatch, cases, targetSha, validateSelection, evidencePolicy } from './chat-completion-testops-contract.mjs'; import { actualContext } from './chat-completion-testops-cli.mjs'; +import { paidWorker } from './chat-completion-testops-host-control.mjs'; + +test('reusable workers bind named environment secrets without inherited or extra credentials', async () => { + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); + const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); + const named = { OPENROUTER_API_KEY: '${{ secrets.OPENROUTER_API_KEY }}', + ANTHROPIC_API_KEY: '${{ secrets.ANTHROPIC_API_KEY }}', ALLURE_TOKEN: '${{ secrets.ALLURE_TOKEN }}' }; + const supported = structuredClone(definition); + for (const id of ['chat', 'chat_external']) supported.jobs[id].secrets = { ...named }; + assert.equal(paidWorker(supported, called), called.jobs.chat); + assert.equal(paidWorker(definition, called), called.jobs.chat); + for (const id of ['chat', 'chat_external']) { + assert.deepEqual(definition.jobs[id].secrets, named); + for (const secrets of [{ ...named, OPENROUTER_API_KEY: '' }, { ...named, EXTRA: '${{ secrets.EXTRA }}' }, + { ...named, ALLURE_TOKEN: '${{ secrets.OTHER_TOKEN }}' }, 'inherit']) { + const mutant = structuredClone(supported); mutant.jobs[id].secrets = secrets; + assert.throws(() => paidWorker(mutant, called), /Workflow caller boundary mismatch/); + } + } +}); function fixture() { const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: 'paperclipai/paperclip', diff --git a/tests/runner-e2e/chat-completion-testops-host-control.mjs b/tests/runner-e2e/chat-completion-testops-host-control.mjs index bae0cdb5fe9..350cb69f54e 100644 --- a/tests/runner-e2e/chat-completion-testops-host-control.mjs +++ b/tests/runner-e2e/chat-completion-testops-host-control.mjs @@ -13,7 +13,8 @@ export function paidWorker(definition, called) { || job.with?.target_sha !== '${{ needs.authorize.outputs.target_sha }}' || job.with?.selection_artifact_id !== '${{ needs.select.outputs.artifact_id }}' || job.with?.selection_sha256 !== '${{ needs.select.outputs.selection_sha256 }}' - || JSON.stringify(job.secrets) !== JSON.stringify({ OPENROUTER_API_KEY: '', ANTHROPIC_API_KEY: '', ALLURE_TOKEN: '' })) { + || JSON.stringify(job.secrets) !== JSON.stringify({ OPENROUTER_API_KEY: '${{ secrets.OPENROUTER_API_KEY }}', + ANTHROPIC_API_KEY: '${{ secrets.ANTHROPIC_API_KEY }}', ALLURE_TOKEN: '${{ secrets.ALLURE_TOKEN }}' })) { throw new Error('Workflow caller boundary mismatch'); } }