From d4b11b2024050ec389cbc75ce3574e828798a7e2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 09:49:07 -0600 Subject: [PATCH] test(evals): run Product regression selections from TestOps Reuse the paid full-stack workflow for the standing 20-case Product regression plan. Bind TestOps case identities, original job runs, source revisions and retained verdicts in separate trusted reporting jobs. Preserve existing paid admission and make missing evidence explicitly broken. Co-Authored-By: Paperclip --- .github/workflows/runner-full-stack-e2e.yml | 132 ++++++++++++++++- doc/evals.md | 12 ++ tests/runner-e2e/README.md | 22 +++ ...product-regression-testops-catalog.test.ts | 18 +++ .../product-regression-testops-cli.mjs | 132 +++++++++++++++++ .../product-regression-testops.d.mts | 10 ++ .../runner-e2e/product-regression-testops.mjs | 140 ++++++++++++++++++ .../product-regression-testops.test.mjs | 131 ++++++++++++++++ tests/runner-e2e/run-unit.mjs | 2 +- tests/runner-e2e/vitest.config.ts | 2 +- 10 files changed, 596 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/product-regression-testops-catalog.test.ts create mode 100644 tests/runner-e2e/product-regression-testops-cli.mjs create mode 100644 tests/runner-e2e/product-regression-testops.d.mts create mode 100644 tests/runner-e2e/product-regression-testops.mjs create mode 100644 tests/runner-e2e/product-regression-testops.test.mjs diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 5f469f96c8a..df8358cf991 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -48,6 +48,18 @@ on: description: "Comma-separated full suite.profile.environment.case IDs; exclusive with other selectors" type: string required: false + testops_case: + description: "Product regression TestOps bootstrap: all, case name or case ID; leave empty for normal campaigns" + type: string + required: false + ALLURE_JOB_RUN_ID: + description: "Allure TestOps service input; its requested selection takes precedence" + type: string + required: false + ALLURE_USERNAME: + description: "Allure TestOps service input" + type: string + required: false permissions: contents: read @@ -284,10 +296,66 @@ jobs: retention-days: 30 if-no-files-found: error + testops_selection: + name: Bind Product regression TestOps selection + needs: authorize + if: github.event_name == 'workflow_dispatch' && (inputs.ALLURE_JOB_RUN_ID != '' || inputs.testops_case != '') + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: runner-e2e-testops-reporting + permissions: + contents: read + actions: read + outputs: + ids: ${{ steps.selection.outputs.ids }} + job_run_id: ${{ steps.selection.outputs.job_run_id }} + selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} + launch_url: ${{ steps.selection.outputs.launch_url }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 + with: + allurectl-version: '2.23.1' + allure-endpoint: https://paperclip.testops.cloud + allure-project-id: '1' + allure-token: ${{ secrets.ALLURE_TOKEN }} + - name: Resolve only admitted plan cases before provider jobs + id: selection + env: + SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + EVAL_CADENCE: ${{ inputs.cadence || 'manual' }} + SELECT_ID: ${{ inputs.id }} + SELECT_SUITE: ${{ inputs.suite }} + SELECT_GROUP: ${{ inputs.group }} + SELECT_PROFILE: ${{ inputs.profile }} + SELECT_ENVIRONMENT: ${{ inputs.environment }} + SELECT_CASE: ${{ inputs.case }} + TESTOPS_CASE: ${{ inputs.testops_case }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }} + ALLURE_LAUNCH_NAME: Product regression · ${{ github.run_number }} + ALLURE_LAUNCH_TAGS: product-regression,github-actions + run: node tests/runner-e2e/product-regression-testops-cli.mjs select product-testops/selection.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }} + path: product-testops/selection.json + if-no-files-found: error + retention-days: 30 + catalog: name: Validate catalog and select cells - needs: [authorize, target_lock] - if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') + needs: [authorize, target_lock, testops_selection] + if: >- + always() && needs.authorize.result == 'success' && needs.target_lock.result == 'success' && + (needs.testops_selection.result == 'success' || needs.testops_selection.result == 'skipped') && + (github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')) runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -371,7 +439,7 @@ jobs: SELECT_PROFILE: ${{ inputs.profile }} SELECT_ENVIRONMENT: ${{ inputs.environment }} SELECT_CASE: ${{ inputs.case }} - SELECT_ID: ${{ inputs.id }} + SELECT_ID: ${{ needs.testops_selection.outputs.ids || inputs.id }} MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }} MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }} REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }} @@ -1946,6 +2014,64 @@ jobs: echo "ready=false" >> "$GITHUB_OUTPUT" fi + publish_testops: + name: Publish bound Product regression results to TestOps + needs: [authorize, testops_selection, catalog, report] + if: always() && !cancelled() && needs.testops_selection.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: runner-e2e-testops-reporting + permissions: + contents: read + actions: read + env: + SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }} + EVAL_CADENCE: manual + ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }} + ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }} + ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure + REPORT_DIR: ${{ github.workspace }}/product-report + EXPORT_SUMMARY: ${{ github.workspace }}/product-regression-export.json + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Only trusted workflow code sees the reporting token. Candidate + # reports are parsed as data and never loaded as executable modules. + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }} + path: product-testops + - name: Download retained full-stack report + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} + path: product-report + - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 + with: + allurectl-version: '2.23.1' + allure-endpoint: https://paperclip.testops.cloud + allure-project-id: '1' + allure-token: ${{ secrets.ALLURE_TOKEN }} + - name: Project original verdicts and explicitly report missing results + id: project + run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json + - name: Upload only the bound result projection + if: steps.project.outcome == 'success' + run: allurectl upload --validate-json "$ALLURE_RESULTS" + - name: Verify server-side case identities, completeness and verdicts + if: steps.project.outcome == 'success' + run: node tests/runner-e2e/product-regression-testops-cli.mjs verify product-testops/selection.json + - name: Finish the original TestOps job run + if: always() + run: allurectl job-run stop + publish_history: name: Publish S3 history and Pages bundle with declared screenshots needs: [authorize, catalog, report] diff --git a/doc/evals.md b/doc/evals.md index 8db280a6219..8f2ec714b82 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,17 @@ # Paperclip evaluation guide +The optional Product regression TestOps lane in `runner-full-stack-e2e.yml` +maps the standing 20-case plan to existing isolated Product E2E cells: native +Codex/local, with native OpenCode/local for the task-workspaces case. A manual +`testops_case` bootstrap accepts `all`, a case name or its TestOps ID. TestOps +dispatch supplies `ALLURE_JOB_RUN_ID`; its exact case IDs and full names take +precedence. This lane preserves the existing paid admission, fixture, model, +timeout, assertion, cleanup and retained-report paths. Reporting credentials +stay in trusted default-branch jobs. Missing evidence becomes a broken result. +Subset runs do not qualify the full plan or the historical 355-row snapshot. +Catalog and projection tests are not live qualification; retain the fresh +TestOps launch and GitHub run when reporting acceptance. + The explicit deployment-targeted [saved agent instructions journey](../tests/runner-e2e/product-journeys/AGENT-SETTINGS.md) checks an owned agent’s ordinary UI save, persisted instruction bytes, one current-run output, and restoration. The task request omits its instruction-only reference. It reuses the existing Product E2E target and report path; local controls do not establish a staging pass. The explicit-only [verified API artifact transfer](../tests/runner-e2e/README.md#verified-api-artifact-transfer) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b7c0a0110b9..c1cb765f0b5 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1591,6 +1591,28 @@ gh workflow run runner-full-stack-e2e.yml \ -f id=core-compatibility.runner-codex.local.message-marker ``` +The same trusted workflow can run the standing **Product regression — core +workflows** TestOps plan. Bootstrap one actual case with `-f testops_case=66` +(message-marker), or select the complete plan with `-f testops_case=all`. +Leave the ordinary `id`, `suite`, `group`, `profile`, `environment` and `case` +selectors empty in this mode. TestOps dispatch supplies `ALLURE_JOB_RUN_ID` and +`ALLURE_USERNAME`; the requested test-plan selection overrides the bootstrap +case. All 20 imported case IDs and full names remain stable. The lane uses the +existing native Codex/local cells and the existing native OpenCode/local +task-workspaces cell. It does not qualify the historical 355-result snapshot. + +The protected `runner-e2e-testops-reporting` environment supplies `ALLURE_TOKEN`. +Only default-branch selection and reporting code receives this token. The +ordinary paid provider jobs, their authorization and their evidence paths stay +the same. Selection binds the job run, launch, source and workflow revisions. +Reporting parses the retained normalized campaign, publishes original attempts +and assertions, and makes an absent result explicitly broken. It verifies the +uploaded case IDs and verdicts before finishing the original TestOps job run. +No bootstrap or synthetic control result contributes to the pass count. Tag +only a complete 20-case attempt `full-run` and `test-plan-2`; omit `full-run` +from focused reruns. Inspect the linked launch and the retained report for +acceptance. A local projection test does not establish a live product pass. + Create a protected `runner-e2e-paid` GitHub environment, restrict it to the default branch, limit environment administration to trusted maintainers, and store the four provider secrets there. This is a second authorization boundary: diff --git a/tests/runner-e2e/product-regression-testops-catalog.test.ts b/tests/runner-e2e/product-regression-testops-catalog.test.ts new file mode 100644 index 00000000000..aecd0524cdb --- /dev/null +++ b/tests/runner-e2e/product-regression-testops-catalog.test.ts @@ -0,0 +1,18 @@ +import { expect, it } from "vitest"; +import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; +import { productRegressionCases } from "./product-regression-testops.mjs"; + +it("runs every standing Product regression case through its existing full-stack fixture", () => { + const executions = selectRunnerExecutions(parseRunnerSelectors( + productRegressionCases.flatMap(row => ["--id", row.executionId]), + )); + expect(executions).toHaveLength(20); + for (const row of productRegressionCases) { + const execution = executions.find(value => value.id === row.executionId)!; + expect(execution.suite.id).toBe(row.suite); + expect(execution.task.id).toBe(row.name); + expect(execution.profile.id).toBe(row.profile); + expect(execution.environment.id).toBe("local"); + expect(execution.requiredCredentials).not.toContain("CODEX_AUTH_JSON"); + } +}); diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs new file mode 100644 index 00000000000..ac5154a9690 --- /dev/null +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -0,0 +1,132 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { createHash } from 'node:crypto'; +import { appendFile, lstat, mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { testopsClient } from './chat-feature-testops-api.mjs'; +import { endpoint, workflowUrl, selectProductRegression, validateSelection, projectProductResults } from './product-regression-testops.mjs'; + +const execute = promisify(execFile); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const fail = code => { throw new Error(`product_regression_testops_${code}`); }; +const positive = value => /^[1-9][0-9]{0,15}$/.test(String(value ?? '')); + +export function admitProductTestOps(env) { + if (env.GITHUB_REPOSITORY !== 'paperclipai/paperclip' || env.GITHUB_REF !== 'refs/heads/master' + || env.GITHUB_EVENT_NAME !== 'workflow_dispatch' + || env.GITHUB_WORKFLOW_REF !== 'paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master' + || !positive(env.GITHUB_RUN_ID) || !positive(env.GITHUB_RUN_ATTEMPT) + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? '') || !/^[a-f0-9]{40}$/.test(env.SOURCE_SHA ?? '') + || env.ALLURE_ENDPOINT !== endpoint || env.ALLURE_PROJECT_ID !== '1' + || (env.EVAL_CADENCE ?? 'manual') !== 'manual' + || ['SELECT_ID', 'SELECT_SUITE', 'SELECT_GROUP', 'SELECT_PROFILE', 'SELECT_ENVIRONMENT', 'SELECT_CASE'].some(key => env[key])) fail('untrusted_context'); +} + +async function readJson(file, limit) { + const stat = await lstat(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > limit) fail('file_invalid'); + return JSON.parse(await readFile(file, 'utf8')); +} + +export function officialProductContext(stdout) { + if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 16_384) fail('official_context'); + const found = {}; + for (const line of stdout.split(/\r?\n/)) for (const key of ['ALLURE_JOB_RUN_ID', 'ALLURE_LAUNCH_ID']) { + if (!line.startsWith(`${key}=`)) continue; + const value = line.slice(key.length + 1); + if (found[key] || !positive(value)) fail('official_context'); + found[key] = value; + } + if (Object.keys(found).length !== 2) fail('official_context'); + return { jobRunId: found.ALLURE_JOB_RUN_ID, launchId: Number(found.ALLURE_LAUNCH_ID) }; +} + +export async function verifyProductJob(api, context, env) { + const run = await api(`/api/jobrun/${context.jobRunId}`); + if (run.id !== Number(context.jobRunId) || run.launchId !== context.launchId || !positive(run.job?.id) + || run.job.type !== 'github' || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`) fail('job_run_mismatch'); + const [job, launch] = await Promise.all([api(`/api/job/${run.job.id}`), api(`/api/launch/${context.launchId}`)]); + if (job.id !== run.job.id || job.type !== 'github' || job.url !== workflowUrl + || launch.id !== context.launchId || launch.projectId !== 1) fail('job_mismatch'); + return { jobId: job.id, launchId: launch.id }; +} + +export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) { + admitProductTestOps(env); + if (command === 'select') { + const requestedJob = env.ALLURE_JOB_RUN_ID || ''; + if (requestedJob && !positive(requestedJob)) fail('job_run_invalid'); + // Start and stop are separated so the official job run remains open while + // the existing isolated GitHub matrix runs in other jobs. + await exec('allurectl', ['job-run', 'start'], { env, timeout: 60_000, maxBuffer: 32_768 }); + const { stdout } = await exec('allurectl', ['job-run', 'env'], { env, timeout: 30_000, maxBuffer: 16_384 }); + const context = officialProductContext(stdout); + if (requestedJob && requestedJob !== context.jobRunId) fail('job_run_mismatch'); + const boundEnv = { ...env, ALLURE_JOB_RUN_ID: context.jobRunId }; + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + const job = await verifyProductJob(api, context, env); + let plan; + if (requestedJob) { + const planFile = path.join(env.RUNNER_TEMP, 'product-regression-testplan.json'); + await exec('allurectl', ['job-run', 'plan', '--job-run-id', context.jobRunId, '--output-file', planFile], + { env: boundEnv, timeout: 30_000, maxBuffer: 32_768 }); + plan = await readJson(planFile, 65_536); + } + const cases = selectProductRegression(plan, env.TESTOPS_CASE || 'all', Boolean(requestedJob)); + const selection = { schema: 'paperclip.product-regression-testops-selection/v1', + runId: env.GITHUB_RUN_ID, attempt: env.GITHUB_RUN_ATTEMPT, workflowSha: env.GITHUB_SHA, + targetSha: env.SOURCE_SHA, ...context, ...job, cases }; + validateSelection(selection, env); + const bytes = Buffer.from(JSON.stringify(selection, null, 2)); + await mkdir(path.dirname(argument), { recursive: true }); + await writeFile(argument, bytes, { flag: 'wx', mode: 0o600 }); + await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\njob_run_id=${context.jobRunId}\nselection_sha256=${hash(bytes)}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`); + return 0; + } + if (!['project', 'verify'].includes(command)) fail('command_invalid'); + const bytes = await readFile(argument); + if (bytes.length > 65_536 || hash(bytes) !== env.SELECTION_SHA256) fail('selection_hash'); + const selection = validateSelection(JSON.parse(bytes), env); + if (env.ALLURE_JOB_RUN_ID !== selection.jobRunId) fail('job_run_mismatch'); + if (command === 'project') { + let campaign; + try { campaign = await readJson(path.join(env.REPORT_DIR, 'normalized', 'normalized-results.json'), 20 * 1024 * 1024); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + const { records, summary } = projectProductResults(campaign, selection, env.SOURCE_SHA); + await mkdir(env.ALLURE_RESULTS, { recursive: false, mode: 0o700 }); + for (const row of records) await writeFile(path.join(env.ALLURE_RESULTS, row.file), JSON.stringify(row.value), { flag: 'wx', mode: 0o600 }); + await writeFile(env.EXPORT_SUMMARY, JSON.stringify(summary), { flag: 'wx', mode: 0o600 }); + await appendFile(env.GITHUB_STEP_SUMMARY, `### [Product regression TestOps results](${endpoint}/launch/${selection.launchId})\n\n${summary.passed}/${summary.selected} passed; ${summary.failed} failed; ${summary.broken} broken. ${summary.fullPlan ? 'Complete 20-case plan selection.' : 'Targeted selection; not a full-plan verdict.'}\n`); + return 0; + } + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + await verifyProductJob(api, selection, env); + const summary = await readJson(env.EXPORT_SUMMARY, 16_384); + // Check server-side identity and count. A successful upload is not a pass. + let observed; + for (let attempt = 0; attempt < 12; attempt++) { + observed = await api(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`); + if (!Array.isArray(observed?.content) || observed.totalPages > 1 || observed.totalElements !== observed.content.length) fail('result_page_invalid'); + const actual = observed.content; + if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) break; + if (attempt === 11) fail('upload_incomplete'); + await new Promise(resolve => setTimeout(resolve, 2_000)); + } + const details = await Promise.all(observed.content.map(row => api(`/api/testresult/${row.id}`))); + if (new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => { + const selected = selection.cases.find(value => value.id === row.testCaseId); + return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId + || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status?.name); + })) fail('uploaded_identity_mismatch'); + const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status.name === status).length])); + if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch'); + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + productTestOps(process.argv[2], process.argv[3]).then(code => { process.exitCode = code; }).catch(error => { + const safe = /^product_regression_testops_[a-z_]+$/.test(error.message) ? error.message : 'product_regression_testops_failed'; + process.stderr.write(`${safe}\n`); process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/product-regression-testops.d.mts b/tests/runner-e2e/product-regression-testops.d.mts new file mode 100644 index 00000000000..097acf7e860 --- /dev/null +++ b/tests/runner-e2e/product-regression-testops.d.mts @@ -0,0 +1,10 @@ +export interface ProductRegressionCase { + id: number; + suite: string; + name: string; + profile: string; + fullName: string; + executionId: string; +} + +export const productRegressionCases: readonly ProductRegressionCase[]; diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs new file mode 100644 index 00000000000..f572b520316 --- /dev/null +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -0,0 +1,140 @@ +import { createHash, randomUUID } from 'node:crypto'; + +// These are the standing, reviewed TestOps plan-2 identities. A case selects one +// existing full-stack cell; it does not change the prompt, matcher or runtime. +export const productRegressionCases = [ + [8, 'core-compatibility', 'plan-revise-accept'], + [9, 'configured-repository', 'repository-output'], + [30, 'core-compatibility', 'ask-question'], + [32, 'first-task', 'reject-no-execution'], + [35, 'agent-chat', 'reassign-task'], + [45, 'agent-chat', 'multi-repository'], + [54, 'task-workspaces', 'task-directory-repository-resume', 'runner-opencode'], + [60, 'first-task', 'task-card-accept'], + [61, 'continuation', 'completed-action-resume'], + [62, 'agent-chat', 'clarify-reuse'], + [66, 'core-compatibility', 'message-marker'], + [68, 'continuation', 'revision-preserves-approval'], + [112, 'agent-chat', 'create-backlog'], + [123, 'first-task', 'revise-accept'], + [134, 'first-task', 'clarify-propose-accept'], + [140, 'agent-chat', 'stop-new-resume'], + [145, 'agent-chat', 'continuity-restart'], + [159, 'continuation', 'clarification-not-approval'], + [164, 'agent-chat', 'plan-handoff'], + [168, 'agent-chat', 'new-session'], +].map(([id, suite, name, profile = 'runner-codex']) => Object.freeze({ + id, suite, name, profile, + fullName: `paperclip.Product E2E.${suite}.${name}`, + executionId: `${suite}.${profile}.local.${name}`, +})); + +const fail = code => { throw new Error(`product_regression_testops_${code}`); }; +const sha = value => createHash('sha256').update(value).digest('hex'); +const identifier = value => typeof value === 'string' && /^[a-zA-Z0-9._:-]{1,200}$/.test(value); +const modelIdentifier = value => typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,199}$/.test(value); +export const endpoint = 'https://paperclip.testops.cloud'; +export const workflowUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/runner-full-stack-e2e.yml'; + +export function selectProductRegression(plan, manual = 'all', fromTestOps = false) { + if (fromTestOps && !plan) fail('plan_required'); + if (plan) { + if (plan.version !== '1.0' || !Array.isArray(plan.tests) || !plan.tests.length || plan.tests.length > productRegressionCases.length) fail('plan_invalid'); + const selected = plan.tests.map(test => { + const row = productRegressionCases.find(value => String(value.id) === String(test.id)); + if (!row || test.selector !== row.fullName) fail('case_mismatch'); + return row; + }); + if (new Set(selected.map(row => row.id)).size !== selected.length) fail('duplicate_case'); + return selected.sort((a, b) => a.id - b.id); + } + if (manual === 'all') return [...productRegressionCases]; + const selected = productRegressionCases.find(row => row.name === manual || String(row.id) === manual); + if (!selected) fail('manual_selection_invalid'); + return [selected]; +} + +export function validateSelection(value, env) { + if (value?.schema !== 'paperclip.product-regression-testops-selection/v1' + || value.runId !== env.GITHUB_RUN_ID || value.attempt !== env.GITHUB_RUN_ATTEMPT + || value.workflowSha !== env.GITHUB_SHA || !/^[a-f0-9]{40}$/.test(value.workflowSha) + || value.targetSha !== env.SOURCE_SHA || !/^[a-f0-9]{40}$/.test(value.targetSha) + || !/^[1-9][0-9]{0,15}$/.test(value.jobRunId) || !Array.isArray(value.cases) + || !value.cases.length || value.cases.length > productRegressionCases.length) fail('selection_binding'); + const selected = selectProductRegression({ version: '1.0', tests: value.cases.map(row => ({ id: row.id, selector: row.fullName })) }, 'all', true); + if (JSON.stringify(selected) !== JSON.stringify(value.cases)) fail('selection_binding'); + return value; +} + +export function projectProductResults(campaign, selection, sourceSha) { + if (!/^[a-f0-9]{40}$/.test(sourceSha)) fail('source_missing'); + if (campaign && (campaign.schema !== 'paperclip.runner-e2e.campaign/v2' + || campaign.campaignId !== `gha-${selection.runId}-${selection.attempt}` || campaign.source?.sha !== sourceSha + || !Array.isArray(campaign.results) || !Array.isArray(campaign.expected) + || JSON.stringify([...campaign.expected].sort()) !== JSON.stringify(selection.cases.map(row => row.executionId).sort()))) fail('report_binding'); + const rows = campaign?.results ?? []; + if (rows.length > selection.cases.length * 10 || rows.some(row => !selection.cases.some(value => value.executionId === row.executionId))) fail('unexpected_result'); + const records = [], summaries = []; + for (const selected of selection.cases) { + const attempts = rows.filter(row => row.executionId === selected.executionId).sort((a, b) => a.attempt - b.attempt); + if (new Set(attempts.map(row => row.attempt)).size !== attempts.length) fail('duplicate_attempt'); + if (!attempts.length) attempts.push(null); + for (const result of attempts) { + const missingArtifact = result?.attempt === 0 && result.status === 'failed' + && result.failureClass === 'permanent_infrastructure' && result.cleanup === 'not_started' + && result.error === 'No result artifact was uploaded' && result.matcherResults === undefined; + if (result && ((!missingArtifact && result.source?.sha !== sourceSha) + || (missingArtifact && result.source?.sha != null && result.source.sha !== sourceSha) + || result.suiteId !== selected.suite || result.caseId !== selected.name + || result.profileId !== selected.profile || result.environmentId !== 'local' + || !Number.isSafeInteger(result.attempt) || result.attempt < (missingArtifact ? 0 : 1) || result.attempt > 10 + || !['passed', 'failed'].includes(result.status) || (!missingArtifact && !Array.isArray(result.matcherResults)))) fail('result_binding'); + const checks = result?.matcherResults ?? []; + if (checks.length > 300 || checks.some(check => typeof check.passed !== 'boolean')) fail('checks_invalid'); + const passed = result?.status === 'passed' && result.cleanup === 'passed' && checks.length > 0 && checks.every(check => check.passed); + const infrastructure = !result || ['permanent_infrastructure', 'transient_infrastructure', 'setup_blocked'].includes(result.failureClass); + const status = passed ? 'passed' : infrastructure ? 'broken' : 'failed'; + const provenance = { + schema: 'paperclip.product-regression-testops-result/v1', caseId: selected.id, + executionId: selected.executionId, workflowRunId: selection.runId, workflowAttempt: selection.attempt, + workflowSha: selection.workflowSha, sourceSha, status, originalStatus: result?.status ?? null, + attempt: result?.attempt ?? null, cleanup: result?.cleanup ?? 'not_started', + profile: selected.profile, environment: 'local', + model: modelIdentifier(result?.model) ? result.model : null, + suiteDefinitionHash: /^[a-f0-9]{64}$/.test(result?.suiteDefinitionHash ?? '') ? result.suiteDefinitionHash : null, + failureClass: identifier(result?.failureClass) ? result.failureClass : null, + durationMs: Number.isFinite(result?.durationMs) && result.durationMs >= 0 ? result.durationMs : null, + checks: checks.map(check => ({ + name: identifier(check.matcher?.path) ? check.matcher.path : identifier(check.matcher?.kind) ? check.matcher.kind : 'assertion', + passed: check.passed, + })), + note: result && !missingArtifact ? 'Original full-stack assertions, attempts and diagnostics remain in the linked workflow report.' + : 'No bound full-stack result was retained. This case is unverified and cannot pass.', + }; + const uuid = randomUUID(), attachment = `${randomUUID()}-attachment.json`; + const value = { + uuid, name: selected.name, fullName: selected.fullName, testCaseId: sha(selected.fullName), + historyId: sha(JSON.stringify([selected.fullName, selected.profile, provenance.model, 'local'])), + status, stage: 'finished', statusDetails: { message: provenance.note }, + labels: [{ name: 'AS_ID', value: String(selected.id) }, { name: 'parentSuite', value: 'Product E2E' }, + { name: 'suite', value: selected.suite }, { name: 'framework', value: 'Paperclip Evals' }], + parameters: [{ name: 'profile', value: selected.profile }, { name: 'environment', value: 'local / GitHub Actions' }, + { name: 'model', value: provenance.model ?? 'Not recorded' }, { name: 'sourceSha', value: sourceSha }], + links: [{ name: 'Full-stack execution and retained diagnostics', type: 'custom', + url: `https://github.com/paperclipai/paperclip/actions/runs/${selection.runId}` }, + { name: 'Evaluated source', type: 'custom', url: `https://github.com/paperclipai/paperclip/commit/${sourceSha}` }], + attachments: [{ name: 'Bound execution provenance and assertions', source: attachment, type: 'application/json' }], + steps: provenance.checks.map(check => ({ name: check.name, status: check.passed ? 'passed' : 'failed', stage: 'finished' })), + }; + if (result && !missingArtifact) for (const [key, date] of [['start', result.startedAt], ['stop', result.finishedAt]]) { + const timestamp = Date.parse(date); + if (Number.isFinite(timestamp)) value[key] = timestamp; + } + records.push({ file: `${uuid}-result.json`, value }, { file: attachment, value: provenance }); + if (result === attempts.at(-1)) summaries.push({ caseId: selected.id, status }); + } + } + return { records, summary: { selected: selection.cases.length, passed: summaries.filter(row => row.status === 'passed').length, + failed: summaries.filter(row => row.status === 'failed').length, broken: summaries.filter(row => row.status === 'broken').length, + fullPlan: selection.cases.length === productRegressionCases.length } }; +} diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs new file mode 100644 index 00000000000..8b1f40d6de9 --- /dev/null +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -0,0 +1,131 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl } from './product-regression-testops.mjs'; +import { admitProductTestOps, officialProductContext, verifyProductJob } from './product-regression-testops-cli.mjs'; + +const workflowSha = 'a'.repeat(40), targetSha = 'b'.repeat(40); +const env = { + GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master', + GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: workflowSha, SOURCE_SHA: targetSha, + ALLURE_ENDPOINT: 'https://paperclip.testops.cloud', ALLURE_PROJECT_ID: '1', +}; +const selected = productRegressionCases.find(row => row.id === 66); +const selection = cases => ({ schema: 'paperclip.product-regression-testops-selection/v1', runId: '123', attempt: '1', + workflowSha, targetSha, jobRunId: '456', launchId: 789, cases }); +const result = (overrides = {}) => ({ + executionId: selected.executionId, suiteId: selected.suite, caseId: selected.name, + profileId: selected.profile, environmentId: 'local', source: { sha: targetSha }, attempt: 1, + status: 'passed', cleanup: 'passed', model: 'gpt-6.1-sol', suiteDefinitionHash: 'c'.repeat(64), + matcherResults: [{ matcher: { kind: 'json_path', path: 'observed.outputVerified', expected: true }, passed: true }], + ...overrides, +}); +const campaign = (results = [result()]) => ({ schema: 'paperclip.runner-e2e.campaign/v2', campaignId: 'gha-123-1', + source: { sha: targetSha }, expected: [selected.executionId], results }); +const allPlan = { version: '1.0', tests: productRegressionCases.map(row => ({ id: String(row.id), selector: row.fullName })) }; + +test('standing plan has 20 unique imported identities and one existing local cell per case', () => { + assert.equal(productRegressionCases.length, 20); + assert.equal(new Set(productRegressionCases.map(row => row.id)).size, 20); + assert.equal(new Set(productRegressionCases.map(row => row.executionId)).size, 20); + assert.deepEqual(productRegressionCases.filter(row => row.profile !== 'runner-codex').map(row => row.id), [54]); + assert.equal(selectProductRegression(allPlan, 'message-marker', true).length, 20); + assert.equal(selectProductRegression(undefined, 'message-marker')[0].id, 66); +}); + +test('a TestOps subset selects only its exact original case; unknown, duplicate and mismatched identities fail', () => { + const subset = { version: '1.0', tests: [{ id: '66', selector: selected.fullName }] }; + assert.deepEqual(selectProductRegression(subset, 'all', true), [selected]); + for (const invalid of [undefined, { version: '1.0', tests: [] }, + { version: '1.0', tests: [{ id: '66', selector: productRegressionCases[0].fullName }] }, + { version: '1.0', tests: [...subset.tests, ...subset.tests] }, + { version: '1.0', tests: [{ id: '9999', selector: selected.fullName }] }]) { + assert.throws(() => selectProductRegression(invalid, 'all', true)); + } +}); + +test('selection authority cannot enter through an untrusted workflow, foreign ref, mixed selector or recurring mode', () => { + admitProductTestOps(env); + for (const mutation of [{ GITHUB_REF: 'refs/heads/candidate' }, { GITHUB_EVENT_NAME: 'pull_request_target' }, + { GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/other.yml@refs/heads/master' }, + { ALLURE_ENDPOINT: 'https://attacker.invalid' }, { SELECT_SUITE: 'core-compatibility' }, + { EVAL_CADENCE: 'daily' }, { SOURCE_SHA: '' }]) assert.throws(() => admitProductTestOps({ ...env, ...mutation })); + validateSelection(selection([selected]), env); + assert.throws(() => validateSelection(selection([selected]), { ...env, GITHUB_RUN_ID: '124' })); + assert.throws(() => validateSelection(selection([selected]), { ...env, SOURCE_SHA: workflowSha })); +}); + +test('official environment parsing keeps only bound IDs and rejects duplicate or malformed context', () => { + assert.deepEqual(officialProductContext('ALLURE_TOKEN=private\nALLURE_JOB_RUN_ID=456\nALLURE_LAUNCH_ID=789\n'), { jobRunId: '456', launchId: 789 }); + for (const stdout of ['ALLURE_JOB_RUN_ID=456\n', 'ALLURE_JOB_RUN_ID=456\nALLURE_JOB_RUN_ID=456\nALLURE_LAUNCH_ID=789', + 'ALLURE_JOB_RUN_ID=456\nALLURE_LAUNCH_ID=not-an-id']) assert.throws(() => officialProductContext(stdout)); +}); + +test('reported pass requires real assertions, successful cleanup and exact current source', () => { + const projected = projectProductResults(campaign(), selection([selected]), targetSha); + assert.equal(projected.summary.passed, 1); + assert.equal(projected.records[0].value.labels.find(row => row.name === 'AS_ID').value, '66'); + assert.equal(projected.records[0].value.fullName, selected.fullName); + for (const partial of [{ cleanup: 'failed' }, { matcherResults: [] }, + { matcherResults: [{ matcher: { kind: 'json_path', path: 'output' }, passed: false }] }]) { + assert.equal(projectProductResults(campaign([result(partial)]), selection([selected]), targetSha).summary.passed, 0); + } + assert.throws(() => projectProductResults(campaign([result({ source: { sha: workflowSha } })]), selection([selected]), targetSha)); + assert.throws(() => projectProductResults({ ...campaign(), campaignId: 'gha-older-1' }, selection([selected]), targetSha)); +}); + +test('missing results remain explicit broken cases and cannot improve the denominator', () => { + const projected = projectProductResults(undefined, selection(productRegressionCases), targetSha); + assert.deepEqual(projected.summary, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true }); + assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 20); + const partial = projectProductResults(campaign([]), selection([selected]), targetSha); + assert.equal(partial.summary.broken, 1); + assert.equal(partial.summary.fullPlan, false); + const missing = result({ attempt: 0, status: 'failed', failureClass: 'permanent_infrastructure', cleanup: 'not_started', + error: 'No result artifact was uploaded', matcherResults: undefined, source: undefined }); + assert.equal(projectProductResults(campaign([missing]), selection([selected]), targetSha).summary.broken, 1); + assert.throws(() => projectProductResults(campaign([{ ...missing, status: 'passed' }]), selection([selected]), targetSha)); +}); + +test('original functional and infrastructure failures survive, and earlier failed retries remain in the export', () => { + for (const [failureClass, expected] of [['candidate_failure', 'failed'], ['permanent_infrastructure', 'broken'], ['transient_infrastructure', 'broken']]) { + const projected = projectProductResults(campaign([result({ status: 'failed', failureClass })]), selection([selected]), targetSha); + assert.equal(projected.records[0].value.status, expected); + } + const projected = projectProductResults(campaign([result({ status: 'failed', failureClass: 'candidate_failure' }), result({ attempt: 2 })]), selection([selected]), targetSha); + assert.deepEqual(projected.records.filter(row => row.file.endsWith('-result.json')).map(row => row.value.status), ['failed', 'passed']); + assert.equal(projected.summary.passed, 1); + assert.throws(() => projectProductResults(campaign([result(), result()]), selection([selected]), targetSha)); +}); + +test('the transmission projection excludes raw errors, provider IDs, narratives and unexpected fields', () => { + const dirty = result({ error: 'secret-error', runIds: ['private-provider-session'], detail: 'private-context', + usage: { credential: 'private-token' }, matcherResults: [{ passed: true, detail: 'secret-detail', + matcher: { kind: 'contains', expected: 'fixture-answer-secret', path: 'unsafe narrative with secrets' } }] }); + const text = JSON.stringify(projectProductResults(campaign([dirty]), selection([selected]), targetSha)); + for (const secret of ['secret-error', 'private-provider-session', 'private-context', 'private-token', 'secret-detail', 'fixture-answer-secret', 'unsafe narrative']) { + assert.equal(text.includes(secret), false, secret); + } +}); + +test('a TestOps job run must belong to this workflow, source launch and GitHub execution', async () => { + const values = { '/api/jobrun/456': { id: 456, launchId: 789, job: { id: 50, type: 'github' }, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }, + '/api/job/50': { id: 50, type: 'github', url: workflowUrl }, '/api/launch/789': { id: 789, projectId: 1 } }; + assert.deepEqual(await verifyProductJob(route => values[route], { jobRunId: '456', launchId: 789 }, env), { jobId: 50, launchId: 789 }); + await assert.rejects(verifyProductJob(route => route === '/api/launch/789' ? { id: 789, projectId: 2 } : values[route], { jobRunId: '456', launchId: 789 }, env)); + await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env)); +}); + +test('workflow keeps reporting credentials and trusted code outside candidate/provider jobs', async () => { + const workflow = await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8'); + const selectionJob = workflow.split(' testops_selection:\n')[1].split('\n catalog:\n')[0]; + const reportingJob = workflow.split(' publish_testops:\n')[1].split('\n publish_history:\n')[0]; + for (const block of [selectionJob, reportingJob]) { + assert.match(block, /environment: runner-e2e-testops-reporting/); + assert.match(block, /ref: \$\{\{ github.sha \}\}/); + assert.doesNotMatch(block, /secrets\.(OPENAI|OPENROUTER|ANTHROPIC)|ref: \$\{\{ needs.authorize.outputs.target_sha/); + } + assert.match(workflow, /SELECT_ID: \$\{\{ needs.testops_selection.outputs.ids \|\| inputs.id \}\}/); + assert.doesNotMatch(workflow.slice(0, workflow.indexOf('\njobs:')), /workflow_call:/); +}); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index b0c55f687c6..a9770f296cf 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -20,7 +20,7 @@ delete env.NODE_OPTIONS; for (const [name, command] of [ ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], - ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", + ["Node tests", ["--test", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", "tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs", diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 42707b008d3..bece70afb21 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -7,7 +7,7 @@ export default defineConfig({ globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], + exclude: [...configDefaults.exclude, "product-regression-testops.test.mjs", "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], testTimeout: 30_000, }, });