PaperClipAI/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs

import test from 'node:test';
import assert from 'node:assert/strict';
import { fullCatalogCases, fullCatalogPlanName, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs';
import { admitDispatch, validateSelection, evidencePolicy, catalogWorkflow, catalogDefinition } from './chat-completion-testops-contract.mjs';
import { bindingFromEnv, validatePilotSelection, verifyPilotUpload, verifyPilotCaseUpload, completePilotRegistration } from './chat-completion-testops-api.mjs';
import { executionMatrix, executionPartition, run } from './chat-completion-testops-cli.mjs';
import { mkdtemp, writeFile, readFile, readdir, rm, realpath } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';

const context = { catalog: 'full', runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: catalogTargetSha };
const runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123';
const selected = fullCatalogCases.map((row, index) => ({ ...row, caseId: row.key === 'tasks.create-backlog' ? 388 : 1000 + index,
  registeredFullName: row.aliases[0] ?? row.fullName }));
const compact = row => ({ key: row.key, caseId: row.caseId, registeredFullName: row.registeredFullName });
const binding = { planId: 10, jobId: 11, integrationId: 1, cases: selected.map(compact) };
const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context,
  projectId: 1, planId: 10, jobId: 11, launchId: 12, allureJobRunId: '77', cases: selected };
const env = { CHAT_FULL_CATALOG_TESTOPS_BINDING: JSON.stringify(binding), ALLURE_JOB_RUN_ID: '77' };
const attachments = [
  { name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 100 },
  { name: 'Private evidence receipt', contentType: 'application/json', contentLength: 200 },
];
const exported = selected.map(row => ({ executionId: row.executionId, caseId: row.caseId, status: 'passed', cleanup: 'passed', attachments }));
function api(replace = {}) {
  const calls = [];
  const get = async route => {
    calls.push(route);
    if (Object.hasOwn(replace, route)) return structuredClone(replace[route]);
    let match;
    if ((match = route.match(/^\/api\/testcase\/(\d+)$/))) {
      const row = selected.find(value => value.caseId === Number(match[1]));
      return { id: row.caseId, projectId: 1, fullName: row.registeredFullName, automated: true };
    }
    if (route === '/api/testplan/10') return { id: 10, projectId: 1, name: fullCatalogPlanName,
      testCasesCount: 244, treeSelection: { inverted: false, leafsInclude: selected.map(row => row.caseId) } };
    if (route === '/api/testplan/10/job') return [{ job: { id: 11 }, testCasesCount: 244 }];
    if (route === '/api/job/11') return { id: 11, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-catalog-testops.yml',
      parameters: [{ id: 34, name: 'Branch', defaultValue: 'refs/heads/master' }] };
    if (route === '/api/jobrun/77') return { id: 77, job: { id: 11, type: 'github' }, stage: 'in_progress', launchId: 12, url: runUrl };
    if (route === '/api/launch/12') return { id: 12, projectId: 1, closed: false };
    if (route === '/api/jobrun/77/plan?expected=true') return selected.map(row => ({ id: row.caseId, selector: row.registeredFullName }));
    if ((match = route.match(/^\/api\/testresult\?launchId=12&page=(\d)&size=100$/))) {
      const index = Number(match[1]);
      return { content: selected.slice(index * 100, (index + 1) * 100).map((row, offset) => ({ id: 2000 + index * 100 + offset, testCaseId: row.caseId })),
        totalElements: 244, totalPages: 3, number: index };
    }
    if ((match = route.match(/^\/api\/testresult\/(\d+)$/))) {
      const id = Number(match[1]), row = selected[id - 2000];
      return { id, projectId: 1, launchId: 12, testCaseId: row.caseId, fullName: row.registeredFullName,
        jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], status: 'passed' };
    }
    if (route.startsWith('/api/testresult/attachment?')) return { content: attachments.map((row, index) => ({ ...row, id: 3000 + index,
      missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 };
    throw new Error(`Unexpected route ${route}`);
  };
  return { get, calls };
}

test('the complete 244 mapping uses compact actual IDs and preserves the legacy case388 alias', async () => {
  assert.ok(Buffer.byteLength(env.CHAT_FULL_CATALOG_TESTOPS_BINDING) < 48_000);
  assert.equal(bindingFromEnv(env, context).cases.length, 244);
  const f = api(), value = await validatePilotSelection(f.get, env, context, 99);
  assert.equal(validateSelection(value, context, '77').length, 244);
  assert.equal(value.cases.find(row => row.caseId === 388).registeredFullName, 'paperclip.Product E2E.chat-features.create-backlog');
  assert.equal(value.targetSha, catalogTargetSha);
});

test('selection accepts the exact master workflow URL returned by a build-server refresh', async () => {
  const job = { id: 11, type: 'github', name: 'Chat catalog TestOps',
    url: 'https://github.com/paperclipai/paperclip/blob/master/.github/workflows/chat-catalog-testops.yml',
    parameters: [{ id: 34, name: 'Branch', defaultValue: 'master' }] };
  const refreshed = api({ '/api/job/11': job });
  const value = await validatePilotSelection(refreshed.get, env, context, 99);
  assert.deepEqual(value.cases.map(row => row.caseId), selected.map(row => row.caseId));
  assert.equal(value.targetSha, catalogTargetSha);
  for (const url of [job.url.replace('/master/', '/other-branch/'), job.url.replace('/paperclipai/', '/other-owner/'),
    job.url.replace('chat-catalog-testops.yml', 'chat-catalog-testops-case.yml')]) {
    const wrong = api({ '/api/job/11': { ...job, url } });
    await assert.rejects(validatePilotSelection(wrong.get, env, context, 99), /job_mismatch/);
    assert.equal(wrong.calls.includes('/api/jobrun/77'), false);
  }
});

test('the actual full registration receipt fits one protected variable without a later source change', async () => {
  const prepared = { planId: 10, integrationId: 1, cases: selected.map(row => Object.fromEntries(
    ['key', 'scenario', 'executionId', 'fullName', 'registeredFullName', 'caseId'].map(key => [key, row[key]]))) };
  const f = api();
  const receipt = await completePilotRegistration(f.get, prepared, { jobRunId: '77', launchId: 12 }, context, 99, async () => {});
  assert.ok(Buffer.byteLength(JSON.stringify(receipt.binding)) < 48_000);
  assert.equal(bindingFromEnv({ CHAT_FULL_CATALOG_TESTOPS_BINDING: JSON.stringify(receipt.binding) }, context).cases.length, 244);
  assert.equal(receipt.binding.cases.find(row => row.caseId === 388).registeredFullName, 'paperclip.Product E2E.chat-features.create-backlog');
});

test('every case has a singleton private approval instead of broadening the original eight-cell authority', () => {
  const matrix = executionMatrix(selection, context, '77');
  assert.equal(matrix.include.length, 244);
  for (const row of selected) {
    const policy = evidencePolicy({ ...selection, cases: [row] }, context, '77');
    assert.deepEqual(policy.approvals[0].executionIds, [row.executionId]);
    assert.equal(policy.approvals[0].targetSha, catalogTargetSha);
    const cell = matrix.include.find(value => value.executionId === row.executionId);
    assert.deepEqual(JSON.parse(cell.private_ids), [row.executionId]);
    assert.deepEqual(JSON.parse(cell.private_policy), policy);
    assert.deepEqual(cell.required_credentials, row.requiredCredentials);
  }
});

test('full catalog source admission is independent from the old three-case workflow', () => {
  const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, ref: 'refs/heads/master',
    sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } };
  const dispatchEnv = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name,
    GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: `paperclipai/paperclip/${catalogWorkflow}@refs/heads/master`, GITHUB_SHA: context.workflowSha,
    GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', TESTOPS_ALLOWED_ACTOR_IDS: '[42]' };
  assert.equal(admitDispatch(dispatchEnv, event, 42).targetSha, catalogTargetSha);
  assert.equal(catalogDefinition({}).cases.length, 3);
  assert.throws(() => admitDispatch({ ...dispatchEnv, GITHUB_RUN_ATTEMPT: '2' }, event, 42), /dispatch/);
  assert.throws(() => admitDispatch(dispatchEnv, { ...event, inputs: { operation: 'launch-plan' } }, 42), /dispatch/);
  assert.throws(() => validateSelection({ ...selection, targetSha: 'b'.repeat(40) }, context, '77'), /selection_binding/);
});

test('native and external partitions preserve exact selections, bounded overlapping small runs and the GitHub output bound', () => {
  for (const rows of [selected, selected.filter(row => !row.externalInput).slice(0, 10), selected.slice(0, 3), selected.filter(row => row.externalInput), [selected[0], selected.find(row => row.externalInput)]]) {
    const input = { ...selection, cases: rows }, partition = executionPartition(input, context, '77');
    const merged = [...partition.native.include, ...partition.external.include].map(row => row.executionId);
    assert.equal(new Set(merged).size, rows.length);
    assert.deepEqual(merged.sort(), rows.map(row => row.executionId).sort());
    if (rows.length <= 3) {
      assert.equal(partition.parallelism, rows.some(row => row.externalInput) ? '1' : '2');
      assert.equal(partition.external.include.length, rows.filter(row => row.externalInput).length);
    } else {
      assert.equal(partition.parallelism, '2'); assert.ok(partition.external.include.every(row => row.external));
    }
    assert.ok([partition.matrix, partition.native, partition.external, rows.map(row => row.executionId)]
      .reduce((sum, value) => sum + JSON.stringify(value).length * 2, 0) < 1_000_000);
    for (const cell of merged) assert.ok(partition.matrix.include.find(row => row.executionId === cell).timeout_minutes <= 45);
  }
});

test('full server readback verifies every result across three complete pages', async () => {
  const f = api();
  const value = await verifyPilotUpload(f.get, selection, context, exported);
  assert.equal(value.verification, 'VERIFIED'); assert.equal(value.results.length, 244);
  assert.ok(f.calls.includes('/api/testresult?launchId=12&page=2&size=100'));
});

test('a full ready snapshot fits its read budget while a late snapshot still fails closed', async t => {
  async function replay(latency) {
    let clock = 0, outcome; const pending = [], f = api();
    const verification = verifyPilotUpload(route => new Promise((resolve, reject) => {
      pending.push({ at: clock + latency, complete: () => { f.get(route).then(resolve, reject); } });
    }), selection, context, exported, { now: () => clock });
    verification.then(value => { outcome = { value }; }, error => { outcome = { error }; });
    for (let tick = 0; tick < 600 && !outcome; tick++) {
      await new Promise(resolve => setImmediate(resolve));
      if (outcome) break;
      assert.ok(pending.length > 0, 'the controlled response chain must make progress');
      clock = Math.min(...pending.map(row => row.at));
      const ready = pending.filter(row => row.at <= clock);
      for (const row of ready) { pending.splice(pending.indexOf(row), 1); row.complete(); }
    }
    assert.ok(outcome, 'bounded controlled snapshot replay must settle');
    return { ...outcome, clock };
  }
  const ready = await replay(1000);
  assert.ifError(ready.error);
  assert.equal(ready.value.verification, 'VERIFIED'); assert.equal(ready.value.results.length, 244);
  assert.ok(ready.clock > 120_000);
  const late = await replay(3000);
  assert.match(late.error?.message ?? '', /upload_unverified/);
  t.diagnostic(JSON.stringify({ fixtureOnly: true, selectedCases: 244, readyElapsedMs: ready.clock, lateElapsedMs: late.clock }));
});

test('changing valid page counts retry only within the existing snapshot bounds', async () => {
  async function replay(alwaysChanging, malformed = false) {
    const f = api(); let polls = 0, waits = 0, clock = 0;
    const get = async route => {
      if (route === '/api/jobrun/77') polls++;
      const page = await f.get(route);
      if (route === '/api/testresult?launchId=12&page=0&size=100' && (alwaysChanging || polls === 1)) {
        return { ...page, totalElements: 150, totalPages: 2 };
      }
      if (malformed && route === '/api/testresult?launchId=12&page=1&size=100') return { ...page, totalElements: '244' };
      return page;
    };
    try {
      const value = await verifyPilotUpload(get, selection, context, exported, { now: () => clock,
        wait: async ms => { clock += ms; waits++; } });
      return { value, polls, waits };
    } catch (error) { return { error, polls, waits }; }
  }
  const converged = await replay(false);
  assert.ifError(converged.error);
  assert.equal(converged.value.verification, 'VERIFIED'); assert.equal(converged.value.results.length, 244);
  assert.equal(converged.polls, 2); assert.equal(converged.waits, 1);
  const moving = await replay(true);
  assert.match(moving.error?.message ?? '', /upload_unverified/);
  assert.equal(moving.polls, 12); assert.equal(moving.waits, 11);
  const malformed = await replay(false, true);
  assert.match(malformed.error?.message ?? '', /result_page/);
  assert.equal(malformed.polls, 1); assert.equal(malformed.waits, 0);
});

test('an incomplete last page and a wrong registered selector cannot qualify a full catalog run', async () => {
  const missing = api({ '/api/testresult?launchId=12&page=2&size=100': { content: [], totalElements: 244, totalPages: 3, number: 2 } });
  await assert.rejects(verifyPilotUpload(missing.get, selection, context, exported), /result_page/);
  const wrong = api({ '/api/jobrun/77/plan?expected=true': [{ id: 388, selector: fullCatalogCases[0].fullName }] });
  await assert.rejects(validatePilotSelection(wrong.get, env, context, 99), /foreign_selection/);
});

test('the real bootstrap facade preserves fixed reconciliation failures and discards unknown lookalike messages', async t => {
  for (const unknown of [false, true]) {
    const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'catalog-bootstrap-failure-')));
    let stderr = ''; const capture = t.mock.method(process.stderr, 'write', value => { stderr += value; return true; });
    try {
      const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' },
        ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'bootstrap' } };
      const eventPath = path.join(root, 'event.json'); await writeFile(eventPath, JSON.stringify(event));
      const runtime = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name,
        GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: `paperclipai/paperclip/${catalogWorkflow}@refs/heads/master`, GITHUB_SHA: context.workflowSha,
        GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1',
        GITHUB_EVENT_PATH: eventPath, GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', ALLURE_TOKEN: 'SYNTHETIC_REPORTING_SECRET',
        RUNNER_TEMP: root, GITHUB_WORKSPACE: root, CHAT_TESTOPS_INTEGRATION_ID: '1' };
      const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99,
        head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: catalogWorkflow,
        repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) });
      const fetchImpl = async (url, options) => {
        if (unknown) throw new Error('chat_full_catalog_testops_catalog_page_private_body');
        if (url.endsWith('/api/uaa/oauth/token')) return Response.json({ access_token: 'SYNTHETIC_BEARER_SECRET' });
        assert.equal(options.method, 'GET');
        if (url.endsWith('/api/integration/1/project/1')) return Response.json({ id: 1, projectId: 1, info: { type: 'github' }, disabled: false });
        assert.ok(url.includes('/api/testcase?'));
        return Response.json({ content: [], number: 0, totalPages: 2, totalElements: 0, private: 'SYNTHETIC_PRIVATE_BODY' });
      };
      assert.equal(await run('bootstrap', undefined, runtime, { exec, fetchImpl }), 1);
      const state = path.join(root, 'chat-completion-testops-bootstrap');
      const failure = JSON.parse(await readFile(path.join(state, 'failure.json'), 'utf8'));
      assert.equal(failure.code, unknown ? 'chat_completion_testops_unknown' : 'chat_full_catalog_testops_catalog_page');
      assert.deepEqual(await readdir(state), ['failure.json']);
      assert.doesNotMatch(stderr + JSON.stringify(failure), /SYNTHETIC_|PRIVATE_BODY|catalog_page_private_body/);
    } finally { capture.mock.restore(); await rm(root, { recursive: true, force: true }); }
  }
});

test('a finished case verifies while known sibling rows coexist in the same launch', async () => {
  const f = api();
  const value = await verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]);
  assert.equal(value.verification, 'VERIFIED');
  assert.deepEqual(value.results.map(row => row.caseId), [selected[0].caseId]);
  assert.equal(f.calls.filter(route => /^\/api\/testresult\/\d+$/.test(route)).length, 1);
});

test('partial verification rejects foreign sibling identities before qualifying the current case', async () => {
  const base = api(), first = await base.get('/api/testresult?launchId=12&page=0&size=100');
  first.content[1].testCaseId = 999999;
  const f = api({ '/api/testresult?launchId=12&page=0&size=100': first });
  await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /foreign_result/);
});

test('partial verification retains the exact current-result jobrun binding', async () => {
  const base = api(), wrong = await base.get('/api/testresult/2000'); wrong.jobRun.id = 78;
  const f = api({ '/api/testresult/2000': wrong });
  await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /result_mismatch/);
});

test('small external selections remain in the serial external lane while native overlap stays bounded', () => {
  const external = selected.filter(row => row.externalInput).slice(0, 2);
  assert.equal(external.length, 2);
  for (const rows of [external, [selected.find(row => !row.externalInput), ...external]]) {
    const partition = executionPartition({ ...selection, cases: rows }, context, '77');
    assert.deepEqual(partition.external.include.map(row => row.executionId), external.map(row => row.executionId));
    assert.ok(partition.native.include.every(row => !row.external));
    assert.equal(partition.parallelism, '1');
    assert.equal(partition.native.include.length + partition.external.include.length, rows.length);
  }
});