From d908af01dcd8500220864f22fa40ebd71b159c76 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 09:51:55 -0600 Subject: [PATCH] test(evals): bind official aliases for the full-stack workflow Accept only the named, master blob and independently verified numeric workflow URLs for the same full-stack workflow. Keep foreign workflow and repository aliases rejected. Co-Authored-By: Paperclip --- tests/runner-e2e/product-regression-testops-cli.mjs | 4 ++-- tests/runner-e2e/product-regression-testops.mjs | 6 ++++++ tests/runner-e2e/product-regression-testops.test.mjs | 7 ++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index f0cc35937f5..664b0c5d1f0 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -5,7 +5,7 @@ import { appendFile, lstat, mkdir, readFile, writeFile } from 'node:fs/promises' import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { testopsClient } from './chat-feature-testops-api.mjs'; -import { endpoint, workflowUrl, selectProductRegression, validateSelection, projectProductResults } from './product-regression-testops.mjs'; +import { endpoint, workflowUrls, selectProductRegression, validateSelection, projectProductResults } from './product-regression-testops.mjs'; const execute = promisify(execFile); const hash = bytes => createHash('sha256').update(bytes).digest('hex'); @@ -47,7 +47,7 @@ export async function verifyProductJob(api, context, env) { if (run.id !== Number(context.jobRunId) || run.launchId !== context.launchId || !positive(run.job?.id) || run.job.type !== 'github' || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`) fail('job_run_mismatch'); const [job, launch] = await Promise.all([api(`/api/job/${run.job.id}`), api(`/api/launch/${context.launchId}`)]); - if (job.id !== run.job.id || job.type !== 'github' || job.url !== workflowUrl + if (job.id !== run.job.id || job.type !== 'github' || !workflowUrls.has(job.url) || launch.id !== context.launchId || launch.projectId !== 1) fail('job_mismatch'); return { jobId: job.id, launchId: launch.id }; } diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs index dc4436c4d50..c4500dbd656 100644 --- a/tests/runner-e2e/product-regression-testops.mjs +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -35,6 +35,12 @@ const identifier = value => typeof value === 'string' && /^[a-zA-Z0-9._:-]{1,200 const modelIdentifier = value => typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,199}$/.test(value); export const endpoint = 'https://paperclip.testops.cloud'; export const workflowUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/runner-full-stack-e2e.yml'; +// Official GitHub/Allure job metadata uses these exact aliases for workflow +// 343807492, independently read from GitHub's workflow API before registration. +export const workflowUrls = new Set([workflowUrl, + 'https://github.com/paperclipai/paperclip/blob/master/.github/workflows/runner-full-stack-e2e.yml', + 'https://github.com/paperclipai/paperclip/actions/workflows/343807492', +]); export function selectProductRegression(plan, manual = 'all', fromTestOps = false) { if (fromTestOps && !plan) fail('plan_required'); diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index 746dd16d6e8..4b3aa4a315f 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; -import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl } from './product-regression-testops.mjs'; +import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl, workflowUrls } from './product-regression-testops.mjs'; import { admitProductTestOps, officialProductContext, verifyProductJob, validateUploadedProductResults } from './product-regression-testops-cli.mjs'; const workflowSha = 'a'.repeat(40), targetSha = 'b'.repeat(40); @@ -129,8 +129,13 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e const values = { '/api/jobrun/456': { id: 456, launchId: 789, job: { id: 50, type: 'github' }, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }, '/api/job/50': { id: 50, type: 'github', url: workflowUrl }, '/api/launch/789': { id: 789, projectId: 1 } }; assert.deepEqual(await verifyProductJob(route => values[route], { jobRunId: '456', launchId: 789 }, env), { jobId: 50, launchId: 789 }); + for (const url of workflowUrls) { + assert.deepEqual(await verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url } : values[route], + { jobRunId: '456', launchId: 789 }, env), { jobId: 50, launchId: 789 }); + } await assert.rejects(verifyProductJob(route => route === '/api/launch/789' ? { id: 789, projectId: 2 } : values[route], { jobRunId: '456', launchId: 789 }, env)); await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env)); + await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: workflowUrl.replace('runner-full-stack-e2e', 'runner-protocol-live-evals') } : values[route], { jobRunId: '456', launchId: 789 }, env)); }); test('server verification accepts real Allure string statuses and rejects wrong verdicts or identities', () => {