PaperClipAI/tests/runner-e2e/product-regression-testops-cli.mjs

import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { createHash } from 'node:crypto';
import { appendFile, lstat, mkdir, readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { testopsClient } from './chat-feature-testops-api.mjs';
import { endpoint, workflowUrls, selectProductRegression, validateSelection, projectProductResults } from './product-regression-testops.mjs';

const execute = promisify(execFile);
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const fail = code => { throw new Error(`product_regression_testops_${code}`); };
const positive = value => /^[1-9][0-9]{0,15}$/.test(String(value ?? ''));

export function admitProductTestOps(env) {
  if (env.GITHUB_REPOSITORY !== 'paperclipai/paperclip' || env.GITHUB_REF !== 'refs/heads/master'
      || env.GITHUB_EVENT_NAME !== 'workflow_dispatch'
      || env.GITHUB_WORKFLOW_REF !== 'paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master'
      || !positive(env.GITHUB_RUN_ID) || !positive(env.GITHUB_RUN_ATTEMPT)
      || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? '') || !/^[a-f0-9]{40}$/.test(env.SOURCE_SHA ?? '')
      || env.ALLURE_ENDPOINT !== endpoint || env.ALLURE_PROJECT_ID !== '1'
      || (env.EVAL_CADENCE ?? 'manual') !== 'manual'
      || ['SELECT_ID', 'SELECT_SUITE', 'SELECT_GROUP', 'SELECT_PROFILE', 'SELECT_ENVIRONMENT', 'SELECT_CASE'].some(key => env[key])) fail('untrusted_context');
}

async function readJson(file, limit) {
  const stat = await lstat(file);
  if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > limit) fail('file_invalid');
  return JSON.parse(await readFile(file, 'utf8'));
}

export function officialProductContext(stdout) {
  if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 16_384) fail('official_context');
  const found = {};
  for (const line of stdout.split(/\r?\n/)) for (const key of ['ALLURE_JOB_RUN_ID', 'ALLURE_LAUNCH_ID']) {
    if (!line.startsWith(`${key}=`)) continue;
    const value = line.slice(key.length + 1);
    if (found[key] || !positive(value)) fail('official_context');
    found[key] = value;
  }
  if (Object.keys(found).length !== 2) fail('official_context');
  return { jobRunId: found.ALLURE_JOB_RUN_ID, launchId: Number(found.ALLURE_LAUNCH_ID) };
}

export async function verifyProductJob(api, context, env, allowUnstarted = false) {
  const run = await api(`/api/jobrun/${context.jobRunId}`);
  if (run.id !== Number(context.jobRunId) || run.launchId !== context.launchId || !positive(run.job?.id)
      || run.job.type !== 'github' || (run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`
        && !(allowUnstarted && (run.url == null || run.url === '')))) fail('job_run_mismatch');
  const [job, launch] = await Promise.all([api(`/api/job/${run.job.id}`), api(`/api/launch/${context.launchId}`)]);
  if (job.id !== run.job.id || job.type !== 'github' || !workflowUrls.has(job.url)
      || launch.id !== context.launchId || launch.projectId !== 1) fail('job_mismatch');
  return { jobId: job.id, launchId: launch.id };
}

export function validateUploadedProductResults(details, selection, summary) {
  if (!Array.isArray(summary.cases) || summary.cases.length !== selection.cases.length
      || new Set(summary.cases.map(row => row.caseId)).size !== selection.cases.length
      || summary.cases.some(row => !selection.cases.some(value => value.id === row.caseId)
        || !['passed', 'failed', 'broken'].includes(row.status))) fail('expected_status_invalid');
  if (details.length !== selection.cases.length || new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => {
    const selected = selection.cases.find(value => value.id === row.testCaseId);
    return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId
      || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status);
  })) fail('uploaded_identity_mismatch');
  const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status === status).length]));
  if (Object.entries(counts).some(([status, count]) => summary[status] !== count)
      || details.some(row => summary.cases.find(value => value.caseId === row.testCaseId).status !== row.status)) fail('uploaded_status_mismatch');
}

export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch,
  wait = ms => new Promise(resolve => setTimeout(resolve, ms))) {
  admitProductTestOps(env);
  if (command === 'select') {
    const requestedJob = env.ALLURE_JOB_RUN_ID || '';
    if (requestedJob && !positive(requestedJob)) fail('job_run_invalid');
    // Reject a bad bootstrap selector before opening any run. Before adopting
    // a TestOps dispatch, independently check its pending job and project.
    const manualCases = requestedJob ? undefined : selectProductRegression(undefined, env.TESTOPS_CASE || 'all');
    const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl);
    if (requestedJob) {
      const pending = await api(`/api/jobrun/${requestedJob}`);
      if (!Number.isSafeInteger(pending.launchId) || pending.launchId < 1) fail('job_run_invalid');
      await verifyProductJob(api, { jobRunId: requestedJob, launchId: pending.launchId }, env, true);
    }
    // Start and stop are separated so the official job run remains open while
    // the existing isolated GitHub matrix runs in other jobs.
    // A final always() job uses the same official CI identity even if start
    // times out ambiguously or cancellation interrupts the following reads.
    await appendFile(env.GITHUB_OUTPUT, 'job_run_started=true\n');
    await exec('allurectl', ['job-run', 'start'], { env, timeout: 60_000, maxBuffer: 32_768 });
    const { stdout } = await exec('allurectl', ['job-run', 'env'], { env, timeout: 30_000, maxBuffer: 16_384 });
    const context = officialProductContext(stdout);
    if (requestedJob && requestedJob !== context.jobRunId) fail('job_run_mismatch');
    await appendFile(env.GITHUB_OUTPUT, `job_run_id=${context.jobRunId}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`);
    const boundEnv = { ...env, ALLURE_JOB_RUN_ID: context.jobRunId };
    const job = await verifyProductJob(api, context, env);
    let plan;
    if (requestedJob) {
      const planFile = path.join(env.RUNNER_TEMP, 'product-regression-testplan.json');
      await exec('allurectl', ['job-run', 'plan', '--job-run-id', context.jobRunId, '--output-file', planFile],
        { env: boundEnv, timeout: 30_000, maxBuffer: 32_768 });
      plan = await readJson(planFile, 65_536);
    }
    const cases = manualCases ?? selectProductRegression(plan, env.TESTOPS_CASE || 'all', true);
    const selection = { schema: 'paperclip.product-regression-testops-selection/v1',
      runId: env.GITHUB_RUN_ID, attempt: env.GITHUB_RUN_ATTEMPT, workflowSha: env.GITHUB_SHA,
      targetSha: env.SOURCE_SHA, ...context, ...job, cases };
    validateSelection(selection, env);
    const bytes = Buffer.from(JSON.stringify(selection, null, 2));
    await mkdir(path.dirname(argument), { recursive: true });
    await writeFile(argument, bytes, { flag: 'wx', mode: 0o600 });
    await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\nselection_sha256=${hash(bytes)}\n`);
    return 0;
  }
  if (!['project', 'verify'].includes(command)) fail('command_invalid');
  const bytes = await readFile(argument);
  if (bytes.length > 65_536 || hash(bytes) !== env.SELECTION_SHA256) fail('selection_hash');
  const selection = validateSelection(JSON.parse(bytes), env);
  if (env.ALLURE_JOB_RUN_ID !== selection.jobRunId) fail('job_run_mismatch');
  if (command === 'project') {
    let campaign;
    try { campaign = await readJson(path.join(env.REPORT_DIR, 'normalized', 'normalized-results.json'), 20 * 1024 * 1024); }
    catch (error) { if (error.code !== 'ENOENT') throw error; }
    const { records, summary } = projectProductResults(campaign, selection, env.SOURCE_SHA);
    await mkdir(env.ALLURE_RESULTS, { recursive: false, mode: 0o700 });
    for (const row of records) await writeFile(path.join(env.ALLURE_RESULTS, row.file), JSON.stringify(row.value), { flag: 'wx', mode: 0o600 });
    await writeFile(env.EXPORT_SUMMARY, JSON.stringify(summary), { flag: 'wx', mode: 0o600 });
    await appendFile(env.GITHUB_STEP_SUMMARY, `### [Product regression TestOps results](${endpoint}/launch/${selection.launchId})\n\n${summary.passed}/${summary.selected} passed; ${summary.failed} failed; ${summary.broken} broken. ${summary.fullPlan ? 'Complete 20-case plan selection.' : 'Targeted selection; not a full-plan verdict.'}\n`);
    return 0;
  }
  const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl);
  await verifyProductJob(api, selection, env);
  const summary = await readJson(env.EXPORT_SUMMARY, 16_384);
  // Check server-side identity and count. A successful upload is not a pass.
  let observed, lastVerification;
  for (let attempt = 0; attempt < 12; attempt++) {
    observed = await api(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`);
    if (!Array.isArray(observed?.content) || observed.totalPages > 1 || observed.totalElements !== observed.content.length) fail('result_page_invalid');
    const actual = observed.content;
    if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) {
      const details = await Promise.all(actual.map(row => api(`/api/testresult/${row.id}`)));
      try { validateUploadedProductResults(details, selection, summary); return 0; }
      catch (error) {
        if (!['product_regression_testops_uploaded_identity_mismatch', 'product_regression_testops_uploaded_status_mismatch'].includes(error.message)) throw error;
        lastVerification = error;
      }
    }
    // Planned rows can exist before the asynchronous upload processor settles
    // their run binding and verdict. Observe this one upload, without rerunning
    // the product or accepting unknown/partial results.
    if (attempt === 11) { if (lastVerification) throw lastVerification; fail('upload_incomplete'); }
    await wait(2_000);
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  productTestOps(process.argv[2], process.argv[3]).then(code => { process.exitCode = code; }).catch(error => {
    const safe = /^product_regression_testops_[a-z_]+$/.test(error.message) ? error.message : 'product_regression_testops_failed';
    process.stderr.write(`${safe}\n`); process.exitCode = 1;
  });
}