PaperClipAI/tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs

import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import path from 'node:path';
import os from 'node:os';
import { admitExecutor, admitOrigin, admitArtifact, admitSelection, attachmentProjection, snapshotAndReplay, extractArchive, origin, routes } from './testops-chat-retained-verification.mjs';

const workflow = '.github/workflows/chat-feature-testops-retained-verification.yml';
const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: 'paperclipai/paperclip',
  GITHUB_REF: 'refs/heads/master', GITHUB_WORKFLOW_REF: `paperclipai/paperclip/${workflow}@refs/heads/master`,
  GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '900', GITHUB_SHA: 'a'.repeat(40), GITHUB_ACTOR_ID: '34892728', TESTOPS_ALLOWED_ACTOR_IDS: '[34892728]' };
const repo = { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' };
const event = { repository: repo, ref: 'refs/heads/master', sender: { id: 34892728 }, inputs: {} };
const ownRun = { id: 900, run_attempt: 1, head_sha: env.GITHUB_SHA, head_branch: 'master', event: 'workflow_dispatch', path: workflow,
  workflow_id: 901, repository: repo, actor: { id: 34892728 }, triggering_actor: { id: 34892728 } };
const registered = { id: 901, path: workflow };
const selection = { schema: 'paperclip.chat-feature-testops-selection/v1', runId: origin.runId, attempt: origin.attempt,
  workflowSha: origin.workflowSha, targetSha: origin.targetSha, allureJobRunId: String(origin.jobRunId),
  projectId: 1, planId: 36, jobId: 34, caseId: 388, launchId: 1069, fullName: 'paperclip.Product E2E.chat-features.create-backlog',
  executionId: 'chat-features.runner-opencode-deepseek-1017.local.create-backlog', canonicalScenario: 'tasks.create-backlog' };
const attachmentMetadata = [
  { name: 'Synthetic task and source conversation', contentType: 'image/png', contentLength: 104828 },
  { name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 1443 },
  { name: 'Sanitized Product E2E record and accounting', contentType: 'application/json', contentLength: 9516 },
];
const receipt = { status: 'passed', cleanup: 'passed', caseId: 388, campaignId: `gha-${origin.runId}-1`, count: 1, attachmentMetadata };
function responses() {
  const jobrun = { id: 1031, job: { id: 34, type: 'github' }, launchId: 1069,
    url: `https://github.com/paperclipai/paperclip/actions/runs/${origin.runId}`, stage: 'finished' };
  const { job, ...embedded } = jobrun;
  return [jobrun, { id: 1069, projectId: 1 }, { content: [{ id: 2675, testCaseId: 388 }], totalElements: 1, totalPages: 1 },
    { id: 2675, projectId: 1, launchId: 1069, testCaseId: 388, fullName: selection.fullName, jobRun: embedded, links: [], status: 'passed' },
    { content: attachmentMetadata.map((row, index) => ({ ...row, id: index + 9274, missed: false, fromTestCase: false })), totalElements: 3, totalPages: 1 }];
}

test('retained executor uses its own live identity rather than the originating evaluation identity', () => {
  const actual = admitExecutor(env, event, repo, ownRun, registered);
  assert.equal(actual.runId, '900');
  assert.equal(actual.source, 'a'.repeat(40));
  assert.notEqual(actual.runId, origin.runId);
  assert.notEqual(actual.source, origin.workflowSha);
  assert.equal(admitExecutor(env, { ...event, inputs: undefined }, repo, ownRun, registered).workflowId, 901);
});

test('foreign source, branch, workflow, actor, replay or caller-selected origin cannot acquire executor admission', () => {
  for (const changed of [{ GITHUB_SHA: 'b'.repeat(40) }, { GITHUB_REF: 'refs/heads/feature' }, { GITHUB_RUN_ATTEMPT: '2' },
    { GITHUB_RUN_ID: origin.runId }, { GITHUB_ACTOR_ID: '1' }, { GITHUB_WORKFLOW_REF: 'other' }]) {
    assert.throws(() => admitExecutor({ ...env, ...changed }, event, repo, ownRun, registered), /executor_rejected/);
  }
  assert.throws(() => admitExecutor(env, { ...event, inputs: { origin: 'other' } }, repo, ownRun, registered), /executor_rejected/);
  assert.throws(() => admitExecutor(env, event, repo, { ...ownRun, triggering_actor: { id: 1 } }, registered), /executor_rejected/);
  assert.throws(() => admitExecutor(env, event, { ...repo, default_branch: 'other' }, ownRun, registered), /executor_rejected/);
  assert.throws(() => admitExecutor(env, event, repo, ownRun, { ...registered, id: 902 }), /executor_rejected/);
});

test('fixed originating measurement requires actual workflow/source/attempt and terminal history', () => {
  const original = { ...ownRun, id: Number(origin.runId), run_attempt: 1, workflow_id: origin.workflowId, head_sha: origin.workflowSha,
    path: '.github/workflows/chat-feature-testops.yml', status: 'completed', conclusion: 'failure' };
  admitOrigin(original);
  for (const changed of [{ head_sha: 'b'.repeat(40) }, { run_attempt: 2 }, { workflow_id: 902 }, { head_branch: 'feature' },
    { conclusion: 'success' }, { actor: { id: 1 } }, { status: 'in_progress' }]) assert.throws(() => admitOrigin({ ...original, ...changed }), /origin_rejected/);
});

test('artifact bytes and metadata must bind the exact admitted origin before TestOps authentication', () => {
  const archive = Buffer.from('synthetic archive');
  const expected = { id: 1, name: 'fixed', limit: 1024, sha256: createHash('sha256').update(archive).digest('hex') };
  const actual = { id: 1, name: 'fixed', size_in_bytes: archive.length, expired: false, digest: `sha256:${expected.sha256}`,
    workflow_run: { id: Number(origin.runId), head_sha: origin.workflowSha } };
  admitArtifact(actual, expected, archive);
  for (const changed of [{ id: 2 }, { name: 'other' }, { expired: true }, { digest: 'sha256:wrong' },
    { workflow_run: { id: 2, head_sha: origin.workflowSha } }]) assert.throws(() => admitArtifact({ ...actual, ...changed }, expected, archive), /artifact_rejected/);
  assert.throws(() => admitArtifact(actual, expected, Buffer.from('changed')), /artifact_rejected/);
  assert.throws(() => admitSelection(Buffer.from(JSON.stringify(selection))), /selection_rejected/);
});

test('fixed protected snapshot performs only the five known GETs and replays the unchanged verifier without replacing live environment', async () => {
  const before = { run: process.env.GITHUB_RUN_ID, source: process.env.GITHUB_SHA }, calls = [], values = responses();
  const result = await snapshotAndReplay(async (route, method) => { calls.push([route, method]); return values[routes.indexOf(route)]; }, selection, receipt);
  assert.deepEqual(calls, routes.map(route => [route, 'GET']));
  assert.equal(result.verification.verification, 'VERIFIED');
  assert.equal(result.verification.resultId, 2675);
  assert.equal(result.verification.runId, origin.runId);
  assert.deepEqual({ run: process.env.GITHUB_RUN_ID, source: process.env.GITHUB_SHA }, before);
  assert.equal(result.attachmentRows.rows.every(row => row.idPositiveSafeInteger && row.missed === false && row.fromTestCase === false), true);
});

test('actual three-way attachment row refusal retains the precise current protected flags without raw fields', async () => {
  for (const change of [{ id: '9274' }, { missed: true }, { fromTestCase: true }]) {
    const values = responses();
    Object.assign(values[4].content[0], change, { secret: 'do-not-retain', source: 'unknown-private-url' });
    const result = await snapshotAndReplay(async route => values[routes.indexOf(route)], selection, receipt);
    assert.equal(result.failure.verification.phase, 'attachment_page');
    assert.equal(result.failure.verification.firstFailedPredicate, 'attachment_rows');
    const safe = JSON.stringify(result);
    assert.ok(!safe.includes('do-not-retain') && !safe.includes('unknown-private-url'));
    assert.equal(result.verification, undefined);
    if (typeof change.id === 'string') assert.equal(result.attachmentRows.rows[0].idType, 'string');
    if (change.missed === true) assert.equal(result.attachmentRows.rows[0].missed, true);
    if (change.fromTestCase === true) assert.equal(result.attachmentRows.rows[0].fromTestCase, true);
  }
  const safe = attachmentProjection({ content: [{ id: 1, name: 'raw secret name', contentType: 'raw secret type', missed: 'raw secret flag' }] });
  assert.ok(!JSON.stringify(safe).includes('raw secret'));
  assert.equal(safe.rows[0].missedType, 'string');
});

test('authoritative wrong job, wrong result jobrun and contradictory source remain failures during historical replay', async () => {
  for (const [mutate, expected] of [[values => { values[0].job.id = 35; }, 'chat_testops_jobrun_mismatch'],
    [values => { values[3].jobRun.id = 1032; }, 'chat_testops_upload_mismatch'],
    [values => { values[3].links = [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }]; }, 'chat_testops_upload_mismatch']]) {
    const values = responses(); mutate(values);
    const result = await snapshotAndReplay(async route => values[routes.indexOf(route)], selection, receipt);
    assert.equal(result.verification, undefined);
    assert.equal(result.failure.code, expected);
  }
});

test('real archive extraction admits only the fixed selection file and rejects traversal or symlinks', async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'retained-chat-archive-'));
  try {
    for (const kind of ['valid', 'traversal', 'symlink']) {
      const archive = path.join(root, `${kind}.zip`), target = path.join(root, kind);
      execFileSync('python3', ['-c', "import zipfile,sys; p,k=sys.argv[1:]; z=zipfile.ZipFile(p,'w'); n='testops-selection.json' if k!='traversal' else '../escaped'; i=zipfile.ZipInfo(n); i.external_attr=(0o120777 if k=='symlink' else 0o100600)<<16; z.writestr(i,b'{}'); z.close()", archive, kind]);
      if (kind === 'valid') { await extractArchive(archive, target, 'selection'); assert.equal(await readFile(path.join(target, 'testops-selection.json'), 'utf8'), '{}'); }
      else await assert.rejects(extractArchive(archive, target, 'selection'));
    }
  } finally { await rm(root, { recursive: true, force: true }); }
});

test('manual workflow has no selection inputs or provider/upload path and retains closed receipts only', async () => {
  const yaml = await readFile(new URL('../../.github/workflows/chat-feature-testops-retained-verification.yml', import.meta.url), 'utf8');
  assert.match(yaml, /workflow_dispatch:/);
  assert.ok(!yaml.includes('inputs:') && !yaml.includes('allurectl') && !yaml.includes('OPENROUTER') && !yaml.includes('id-token: write'));
  assert.match(yaml, /environment: runner-e2e-testops-reporting/);
  assert.match(yaml, /ref: \$\{\{ github.sha \}\}/);
  assert.match(yaml, /chat-testops-retained-verification\/retained-verification.json/);
  assert.ok(!yaml.includes('safe-export\n') && !yaml.includes('selection.zip'));
});