import test from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { parse } from 'yaml'; import { admitDispatch, cases, targetSha, validateSelection, evidencePolicy } from './chat-completion-testops-contract.mjs'; import { actualContext } from './chat-completion-testops-cli.mjs'; import { paidWorker } from './chat-completion-testops-host-control.mjs'; test('reusable workers bind named environment secrets without inherited or extra credentials', async () => { const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); const named = { OPENROUTER_API_KEY: '${{ secrets.OPENROUTER_API_KEY }}', ANTHROPIC_API_KEY: '${{ secrets.ANTHROPIC_API_KEY }}', ALLURE_TOKEN: '${{ secrets.ALLURE_TOKEN }}' }; const supported = structuredClone(definition); for (const id of ['chat', 'chat_external']) supported.jobs[id].secrets = { ...named }; assert.equal(paidWorker(supported, called), called.jobs.chat); assert.equal(paidWorker(definition, called), called.jobs.chat); for (const id of ['chat', 'chat_external']) { assert.deepEqual(definition.jobs[id].secrets, named); for (const secrets of [{ ...named, OPENROUTER_API_KEY: '' }, { ...named, EXTRA: '${{ secrets.EXTRA }}' }, { ...named, ALLURE_TOKEN: '${{ secrets.OTHER_TOKEN }}' }, 'inherit']) { const mutant = structuredClone(supported); mutant.jobs[id].secrets = secrets; assert.throws(() => paidWorker(mutant, called), /Workflow caller boundary mismatch/); } } }); function fixture() { const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', GITHUB_ACTOR_ID: '42', TESTOPS_ALLOWED_ACTOR_IDS: '[42]' }; const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, ref: env.GITHUB_REF, sender: { id: 42 }, inputs: { ALLURE_JOB_RUN_ID: '77' } }; const context = admitDispatch(env, event, 42); const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, allureJobRunId: '77', planId: 10, jobId: 11, launchId: 12, cases: cases.map((row, index) => ({ ...row, caseId: 100 + index })) }; return { env, event, context, selection }; } test('actual TestOps selection can execute the three exact cases or a genuine subset', () => { const f = fixture(); assert.equal(validateSelection(f.selection, f.context, '77').length, 3); f.selection.cases = [f.selection.cases[2]]; const policy = evidencePolicy(f.selection, f.context, '77'); assert.deepEqual(policy.approvals[0].executionIds, [cases[2].executionId]); assert.equal(policy.approvals[0].targetSha, targetSha); }); for (const kind of ['foreign-workflow', 'candidate-ref', 'replay', 'unlisted-trigger', 'foreign-repo', 'extra-source-input', 'missing-jobrun']) { test(`rejects ${kind} before any service or provider credential`, () => { const f = fixture(); let trigger = 42; if (kind === 'foreign-workflow') f.env.GITHUB_WORKFLOW_REF = f.env.GITHUB_WORKFLOW_REF.replace('chat-completion', 'chat-feature'); if (kind === 'candidate-ref') f.env.GITHUB_REF = 'refs/heads/codex/candidate'; if (kind === 'replay') f.env.GITHUB_RUN_ATTEMPT = '2'; if (kind === 'unlisted-trigger') trigger = 99; if (kind === 'foreign-repo') f.event.repository.full_name = 'foreign/paperclip'; if (kind === 'extra-source-input') f.event.inputs.targetSha = 'b'.repeat(40); if (kind === 'missing-jobrun') delete f.event.inputs.ALLURE_JOB_RUN_ID; assert.throws(() => admitDispatch(f.env, f.event, trigger)); }); } for (const kind of ['foreign-case', 'duplicate', 'wrong-fullname', 'foreign-source', 'wrong-run', 'wrong-jobrun', 'duplicate-case-id']) { test(`refuses evidence authority for ${kind}`, () => { const f = fixture(); if (kind === 'foreign-case') f.selection.cases[0].executionId = 'chat-features.runner-opencode.local.tasks.delete'; if (kind === 'duplicate') f.selection.cases.push(f.selection.cases[0]); if (kind === 'wrong-fullname') f.selection.cases[0].fullName = cases[1].fullName; if (kind === 'foreign-source') f.selection.targetSha = 'b'.repeat(40); if (kind === 'wrong-run') f.selection.runId = '456'; if (kind === 'wrong-jobrun') f.selection.allureJobRunId = '78'; if (kind === 'duplicate-case-id') f.selection.cases[1].caseId = f.selection.cases[0].caseId; assert.throws(() => evidencePolicy(f.selection, f.context, '77')); }); } test('bootstrap and the fenced plan launcher cannot carry a TestOps execution identity', () => { const f = fixture(); f.event.inputs = { operation: 'bootstrap' }; assert.equal(admitDispatch(f.env, f.event, 42).operation, 'bootstrap'); f.event.inputs.operation = 'launch-plan'; assert.equal(admitDispatch(f.env, f.event, 42).operation, 'launch-plan'); f.event.inputs.ALLURE_JOB_RUN_ID = '77'; assert.throws(() => admitDispatch(f.env, f.event, 42)); }); for (const value of ['00077', '9007199254740992', '12345678901234567']) { test(`dispatch rejects unsafe job identity ${value}`, () => { const f = fixture(); f.event.inputs.ALLURE_JOB_RUN_ID = value; assert.throws(() => admitDispatch(f.env, f.event, 42)); }); } test('dispatch preserves the maximum safe TestOps job identity', () => { const f = fixture(); f.event.inputs.ALLURE_JOB_RUN_ID = '9007199254740991'; assert.equal(admitDispatch(f.env, f.event, 42).operation, 'evaluate'); }); test('the executable guard binds the actual original GitHub run before service or provider access', async () => { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-context-'))); try { const f = fixture(), eventPath = path.join(root, 'event.json'); await writeFile(eventPath, JSON.stringify(f.event)); const env = { ...f.env, GITHUB_EVENT_PATH: eventPath, GH_TOKEN: 'synthetic-gh-token', GITHUB_RUN_NUMBER: '5' }; const run = { id: 123, run_attempt: 1, run_number: 5, workflow_id: 99, head_sha: env.GITHUB_SHA, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-completion-testops.yml', repository: { full_name: 'paperclipai/paperclip', private: false }, actor: { id: 42 }, triggering_actor: { id: 42 } }; let calls = 0; const exec = async (program, args) => { calls++; assert.equal(program, 'gh'); assert.deepEqual(args, ['api', 'repos/paperclipai/paperclip/actions/runs/123']); return { stdout: JSON.stringify(run) }; }; assert.equal((await actualContext(env, exec)).workflowId, 99); assert.equal(calls, 1); for (const field of ['head_sha', 'path', 'head_branch', 'run_attempt', 'run_number', 'workflow_id']) { const other = { ...run, [field]: typeof run[field] === 'number' ? run[field] + 1 : 'wrong' }; if (field === 'workflow_id') other[field] = 0; await assert.rejects(actualContext(env, async () => ({ stdout: JSON.stringify(other) }))); } await assert.rejects(actualContext({ ...env, GITHUB_RUN_ID: '../other' }, () => assert.fail('no request before numeric admission'))); } finally { await rm(root, { recursive: true, force: true }); } }); test('target checkout cannot populate or overwrite the trusted helper path', async () => { const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); const jobs = [definition.jobs.build, called.jobs.chat]; const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); try { const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); git('init'); await writeFile(path.join(root, 'fixture.txt'), 'normal target'); git('add', '.'); git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'normal target'); for (const job of jobs) { const target = job.steps.findIndex(step => ['${{ needs.authorize.outputs.target_sha }}', '${{ inputs.target_sha }}'].includes(step.with?.ref)); const boundary = job.steps.findIndex(step => step.id === 'target_boundary'); const trusted = job.steps.findIndex(step => step.with?.path === '.testops-trusted'); assert.ok(target >= 0 && boundary > target && trusted > boundary, 'Reject reserved target paths, then check out trusted main last'); assert.notEqual(job.steps[target].with.clean, false); execFileSync('bash', ['-euo', 'pipefail', '-c', job.steps[boundary].run], { cwd: root, stdio: 'pipe' }); } await mkdir(path.join(root, '.testops-trusted')); await writeFile(path.join(root, '.testops-trusted', 'helper.js'), 'candidate-controlled helper'); git('add', '.testops-trusted'); git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'reserved path in target'); for (const job of jobs) { const guard = job.steps.find(step => step.id === 'target_boundary'); assert.throws(() => execFileSync('bash', ['-euo', 'pipefail', '-c', guard.run], { cwd: root, stdio: 'pipe' })); } } finally { await rm(root, { recursive: true, force: true }); } });