fix(testops): report retained catalog results with explicit gaps

Preserve authentic original grades when missing retained projections prevent complete reporting. Keep unavailable evidence separate from evaluation results.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -3371,3 +3371,43 @@ It uses `local_trusted` authentication and no model provider. Its result does no
 qualify browser reconnects, active-agent continuity, distinct packaged releases
 or Railway. Keep failed receipts alongside successful attempts; the private
 server logs and config files may contain credentials and are not shareable evidence.
+
+### Report a terminal catalog run from retained evidence
+
+A missing cell projection makes the normal strict aggregate fail. It must not
+silently discard authentic sibling results or turn unavailable evidence into an
+original failed evaluation. For an operator recovery, authenticate the original
+selection, terminal GitHub jobs and exact retained archive keys first. Recover
+available originals through the existing sealed-inventory reader and exporter.
+Do not run agents again or change the original grades.
+
+The aggregate owner has an explicit offline data preparation mode:
+
+```sh
+node --import tsx tests/runner-e2e/chat-full-catalog-testops-merge.ts retained-reporting \
+  <safe-singleton-projection-root> <new-output-directory> \
+  <authenticated-original-selection.json> <retained-disposition.json> <executor-commit-sha>
+```
+
+`RetainedDisposition` in that module defines the closed input contract. It binds
+its origin to the selection, its selection digest to the exact two-space JSON
+serialization, and its terminal-job and custody observations to SHA-256 digests.
+Every missing execution needs an explicit `not_found` observation at its exact
+source-derived archive key. This input records an operator's authenticated
+observations; the offline command does not grant read or publish authority.
+Active jobs, unexplained omissions, duplicates and foreign identities fail.
+
+The command reuses the unchanged strict original exporter. It emits one frozen
+Allure bundle and `reporting-manifest.json`. The coverage count separates graded
+originals from reporting gaps. Gaps are separate DTOs with UNKNOWN original grade,
+checks, native-run count and cleanup. Their TestOps outcome is `broken`, not an
+inferred feature failure or pass. They have no fabricated duration or test steps.
+Only the flat `allure/` JSON files are transport files; the manifest stays outside
+any upload archive. Private originals, screenshots and traces stay private.
+
+Review the immutable payload and its file hashes before an authorized upload to
+the original jobrun. Persist a once-intent before transmission. After an ambiguous
+response, reconcile with GET requests; do not resend or regenerate UUIDs. Final
+coverage must match all selected case IDs and preserve the separate gap count.
+This mode prepares data only. It does not upload, start jobs, reopen launches,
+change credentials, or change evaluation and evidence limits.

tests/runner-e2e/chat-full-catalog-testops-merge.ts

@@ -1,9 +1,10 @@
 import { lstat, mkdir, readFile, readdir, realpath, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { randomUUID } from "node:crypto";
 import { pathToFileURL } from "node:url";
 import { buildRunnerCampaign } from "./history.js";
-import { readPilotExport } from "./chat-completion-testops-results.mjs";
-import { hash, validateSelection } from "./chat-completion-testops-contract.mjs";
+import { readPilotExport, exportPilotAllure } from "./chat-completion-testops-results.mjs";
+import { hash, validateSelection, catalogDefinition } from "./chat-completion-testops-contract.mjs";
 import type { PilotContext, PilotSelection } from "./chat-completion-testops-contract.mjs";
 import type { RunnerE2EResult } from "./types.js";
 import { findSecretLeakInJsonValues } from "./redaction.js";
@@ -49,11 +50,129 @@ export async function mergeCatalogProjections(root: string, output: string, sele
   return binding;
 }
 
+export interface RetainedDisposition {
+  schema: "paperclip.chat-catalog-retained-disposition/v1";
+  origin: PilotContext & { projectId: number; planId: number; jobId: number; launchId: number; allureJobRunId: string };
+  selectionSha256: string;
+  upstream: { status: "completed"; jobsSha256: string };
+  custodySha256: string;
+  gaps: Array<{ executionId: string; archiveKey: string; archiveStatus: "not_found" }>;
+}
+
+/** Report retained measurements without turning missing evidence into a measurement.
+ * This is an offline data transform, not dispatch authority. The operator must
+ * authenticate the selection, terminal jobs and exact-key custody observations
+ * before invoking it. Origin and recovery executor identities remain separate.
+ */
+export async function prepareRetainedCatalogReport(root: string, output: string, selection: PilotSelection, context: PilotContext,
+  disposition: RetainedDisposition, executor: { kind: "operator"; revisionSha: string }) {
+  const selected = validateSelection(selection, context, selection.allureJobRunId);
+  const keys = (value: object, expected: string[]) => Object.keys(value).sort().join() === [...expected].sort().join();
+  const sha = (value: string) => /^[a-f0-9]{64}$/.test(value);
+  const origin = { catalog: context.catalog, runId: context.runId, attempt: context.attempt, workflowSha: context.workflowSha,
+    targetSha: context.targetSha, projectId: selection.projectId, planId: selection.planId, jobId: selection.jobId,
+    launchId: selection.launchId, allureJobRunId: selection.allureJobRunId };
+  if (context.catalog !== "full" || context.targetSha !== catalogDefinition(context).targetSha || context.attempt !== "1"
+    || !/^[1-9][0-9]{0,15}$/.test(context.runId) || !/^[a-f0-9]{40}$/.test(context.workflowSha)
+    || !keys(disposition, ["schema", "origin", "selectionSha256", "upstream", "custodySha256", "gaps"])
+    || disposition.schema !== "paperclip.chat-catalog-retained-disposition/v1"
+    || !keys(disposition.origin, Object.keys(origin)) || Object.entries(origin).some(([key, value]) => disposition.origin[key as keyof typeof origin] !== value)
+    || disposition.selectionSha256 !== hash(JSON.stringify(selection, null, 2))
+    || !keys(disposition.upstream, ["status", "jobsSha256"]) || disposition.upstream.status !== "completed" || !sha(disposition.upstream.jobsSha256)
+    || !sha(disposition.custodySha256) || !Array.isArray(disposition.gaps) || disposition.gaps.length > selected.length
+    || !keys(executor, ["kind", "revisionSha"]) || executor.kind !== "operator" || !/^[a-f0-9]{40}$/.test(executor.revisionSha)) throw new Error("Invalid retained reporting disposition");
+  const gaps = new Set<string>();
+  for (const gap of disposition.gaps) {
+    if (!keys(gap, ["executionId", "archiveKey", "archiveStatus"]) || gaps.has(gap.executionId)
+      || !selected.some(row => row.executionId === gap.executionId) || gap.archiveStatus !== "not_found"
+      || gap.archiveKey !== `runner-e2e-private/bounded-v1/campaigns/gha-${context.runId}-1/manual-cell-${hash(gap.executionId)}.zip`) throw new Error("Invalid reporting gap");
+    gaps.add(gap.executionId);
+  }
+  const stat = await lstat(root);
+  if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(root) !== path.resolve(root)) throw new Error("Invalid projection root");
+  const originals = new Set<string>();
+  for (const name of await readdir(root)) {
+    if (!/^[A-Za-z0-9._-]+$/.test(name)) throw new Error("Invalid projection path");
+    const directory = path.join(root, name), info = await lstat(directory), bindingFile = path.join(directory, "binding.json");
+    if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Invalid projection path");
+    const bindingStat = await lstat(bindingFile);
+    if (!bindingStat.isFile() || bindingStat.isSymbolicLink() || bindingStat.nlink !== 1 || bindingStat.size > 262_144) throw new Error("Invalid projection binding");
+    const binding = JSON.parse(await readFile(bindingFile, "utf8"));
+    const id = binding.executionIds?.length === 1 ? binding.executionIds[0] : undefined;
+    const row = selected.find(value => value.executionId === id);
+    if (!row || originals.has(id) || gaps.has(id)) throw new Error("Duplicate or foreign retained partition");
+    await readPilotExport(directory, context, { ...selection, cases: [row] }); originals.add(id);
+  }
+  if (originals.size + gaps.size !== selected.length) throw new Error("Incomplete retained reporting partition");
+  await mkdir(output, { mode: 0o700 });
+  const originalSelection = { ...selection, cases: selection.cases.filter(row => originals.has(row.executionId)) };
+  const allure = path.join(output, "allure");
+  let exported: Array<{ caseId: number; executionId: string; status: string; cleanup: string; attachments: unknown[] }> = [];
+  if (originals.size) {
+    const merged = path.join(output, "originals");
+    await mergeCatalogProjections(root, merged, originalSelection, context);
+    exported = await exportPilotAllure(merged, allure, context, originalSelection);
+  } else await mkdir(allure, { mode: 0o700 });
+  for (const row of selected.filter(value => gaps.has(value.executionId))) {
+    const dto = { schema: "paperclip.chat-catalog-reporting-gap/v1", outcome: "reporting_gap", originalStatus: null,
+      nativeRunCount: null, checks: null, cleanup: null, doesNotAttestFeatureOutcome: true,
+      executionId: row.executionId, caseId: row.caseId, fullName: row.registeredFullName ?? row.fullName,
+      origin, recoveryExecutor: executor, reason: "retained_archive_unavailable",
+      selectionSha256: disposition.selectionSha256, terminalJobsSha256: disposition.upstream.jobsSha256, custodySha256: disposition.custodySha256 };
+    const attachments = [];
+    for (const [name, value] of [["Reporting gap (original grade UNKNOWN)", dto],
+      ["Retained evidence custody", { schema: "paperclip.chat-catalog-reporting-gap-custody/v1", executionId: row.executionId,
+        archiveStatus: "not_found_at_expected_retained_key", originalEvidenceAvailable: false, doesNotAttestFeatureOutcome: true,
+        custodySha256: disposition.custodySha256 }]] as const) {
+      const source = `${randomUUID()}-attachment.json`;
+      await writeFile(path.join(allure, source), JSON.stringify(value), { flag: "wx", mode: 0o600 });
+      attachments.push({ name, source, type: "application/json" });
+    }
+    const uuid = randomUUID();
+    await writeFile(path.join(allure, `${uuid}-result.json`), JSON.stringify({ uuid, name: row.name,
+      fullName: row.registeredFullName ?? row.fullName,
+      testCaseId: hash(`${JSON.stringify(row.registeredFullName ?? row.fullName)}\n`),
+      historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", "openrouter/deepseek/deepseek-v4.1-flash-1017", "local"])),
+      status: "broken", stage: "finished", statusDetails: { message: "Reporting gap: retained original unavailable. Original evaluation grade is UNKNOWN. No evaluation was rerun." },
+      labels: [{ name: "AS_ID", value: String(row.caseId) }, { name: "parentSuite", value: "Product E2E" },
+        { name: "suite", value: "chat-features" }, { name: "framework", value: "Paperclip Evals" }, { name: "tag", value: "reporting-gap" }],
+      parameters: [{ name: "canonicalScenario", value: row.key }, { name: "originalGrade", value: "UNKNOWN" }],
+      links: [{ name: "Original GitHub Actions run", type: "custom", url: `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` }],
+      steps: [], attachments }), { flag: "wx", mode: 0o600 });
+  }
+  const files = [];
+  for (const name of (await readdir(allure)).sort()) {
+    const bytes = await readFile(path.join(allure, name));
+    if (findSecretLeakInJsonValues(JSON.parse(bytes.toString()), [], { includeShapes: true })) throw new Error("Unsafe reporting payload");
+    files.push({ path: name, bytes: bytes.length, sha256: hash(bytes) });
+  }
+  if (files.length !== selected.length * 3 || files.reduce((sum, row) => sum + row.bytes, 0) > 32 * 1024 * 1024) throw new Error("Invalid reporting payload inventory");
+  const receipt = { schema: "paperclip.chat-catalog-retained-report/v1", origin, recoveryExecutor: executor,
+    selectionSha256: disposition.selectionSha256, dispositionSha256: hash(JSON.stringify(disposition)),
+    coverage: { selected: selected.length, gradedOriginals: originals.size,
+      originalPassed: exported.filter(row => row.status === "passed").length, originalFailed: exported.filter(row => row.status !== "passed").length,
+      reportingGaps: gaps.size },
+    rows: selected.map(row => ({ executionId: row.executionId, caseId: row.caseId,
+      originalStatus: exported.find(value => value.executionId === row.executionId)?.status ?? null,
+      reportingStatus: exported.find(value => value.executionId === row.executionId)?.status ?? "broken" })), files };
+  await writeFile(path.join(output, "reporting-manifest.json"), JSON.stringify(receipt, null, 2), { flag: "wx", mode: 0o600 });
+  return receipt;
+}
+
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  if (process.argv[2] === "retained-reporting") {
+    const [root, output, selectionFile, dispositionFile, executorSha] = process.argv.slice(3);
+    const selection = JSON.parse(await readFile(selectionFile!, "utf8"));
+    const context = { catalog: "full" as const, runId: selection.runId, attempt: selection.attempt,
+      workflowSha: selection.workflowSha, targetSha: selection.targetSha };
+    await prepareRetainedCatalogReport(path.resolve(root!), path.resolve(output!), selection, context,
+      JSON.parse(await readFile(dispositionFile!, "utf8")), { kind: "operator", revisionSha: executorSha! });
+  } else {
   const bytes = await readFile(process.env.TESTOPS_CATALOG_SELECTION!);
   if (hash(bytes) !== process.env.EXPECTED_SELECTION_SHA256) throw new Error("Selection digest mismatch");
   const selection = JSON.parse(bytes.toString("utf8"));
   const context = { catalog: "full" as const, runId: process.env.GITHUB_RUN_ID!, attempt: process.env.GITHUB_RUN_ATTEMPT!,
     workflowSha: process.env.GITHUB_SHA!, targetSha: selection.targetSha };
   await mergeCatalogProjections(path.resolve("catalog-projections"), path.resolve("runner-e2e-testops"), selection, context);
+  }
 }

tests/runner-e2e/chat-full-catalog-testops-retained.test.ts

@@ -0,0 +1,53 @@
+import { it as test, afterEach } from 'vitest';
+import assert from 'node:assert/strict';
+import {mkdtemp,mkdir,readFile,readdir,writeFile,rm,realpath} from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import * as aggregate from './chat-full-catalog-testops-merge.js';
+import {hash,model} from './chat-completion-testops-contract.mjs';
+import {fullCatalogCases,catalogTargetSha} from './chat-full-catalog-testops-manifest.mjs';
+const roots:string[]=[];
+afterEach(async()=>{await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));});
+const context={catalog:'full' as const,runId:'123',attempt:'1',workflowSha:'a'.repeat(40),targetSha:catalogTargetSha};
+async function fixture(t:any){
+ const root=await realpath(await mkdtemp(path.join(os.tmpdir(),'chat-retained-')));roots.push(root);
+ const input=path.join(root,'inputs');await mkdir(input);
+ const cases=fullCatalogCases.slice(0,3).map((row,i)=>({...row,caseId:100+i}));
+ const selection={schema:'paperclip.chat-completion-testops-selection/v1',...context,projectId:1,planId:102,jobId:100,launchId:12,allureJobRunId:'77',cases};
+ for(const [index,row] of cases.slice(0,2).entries()){
+  const dir=path.join(input,row.key);await mkdir(dir);
+  const passed=index===0;
+  const result={schema:'paperclip.runner-e2e.result/v2',suiteDefinitionHash:'c'.repeat(64),startedAt:'2026-11-01T00:00:00Z',finishedAt:'2026-11-01T00:00:01Z',durationMs:1000,executionId:row.executionId,suiteId:'chat-features',caseId:row.scenario,profileId:'runner-opencode',provider:'opencode',environmentId:'local',model,runtimeMode:'native',attempt:1,source:{sha:catalogTargetSha,ref:null,workflowRunUrl:'https://github.com/paperclipai/paperclip/actions/runs/123'},status:passed?'passed':'failed',cleanup:passed?'passed':'failed',runIds:passed?Array.from({length:row.minimumRuns},(_,i)=>`owned-${i}`):[],matcherResults:passed?Array.from({length:row.checks??2},(_,i)=>({matcher:{kind:'json_path',path:`fixture.check${i}`,expected:true},passed:true,detail:'Retained independent check'})):[],screenshots:[]};
+  const files=new Map([['normalized-results.json',Buffer.from(JSON.stringify({schema:'paperclip.runner-e2e.campaign/v2',campaignId:'gha-123-1',retries:0,results:[result]}))],['junit.xml',Buffer.from('<testsuite/>')]]);
+  for(const [name,data] of files)await writeFile(path.join(dir,name),data);
+  await writeFile(path.join(dir,'binding.json'),JSON.stringify({schema:'paperclip.chat-completion-testops-export/v1',...context,campaignId:'gha-123-1',executionIds:[row.executionId],files:[...files].map(([name,data])=>({path:name,bytes:data.length,sha256:hash(data)})),privateEvidence:[{executionId:row.executionId,requiredEvidenceComplete:passed,finalScreenshot:passed?{sha256:'e'.repeat(64),bytes:24}:null}]}));
+ }
+ const disposition={schema:'paperclip.chat-catalog-retained-disposition/v1',origin:{...context,projectId:1,planId:102,jobId:100,launchId:12,allureJobRunId:'77'},selectionSha256:hash(JSON.stringify(selection,null,2)),upstream:{status:'completed',jobsSha256:'b'.repeat(64)},custodySha256:'c'.repeat(64),gaps:[{executionId:cases[2].executionId,archiveKey:`runner-e2e-private/bounded-v1/campaigns/gha-123-1/manual-cell-${hash(cases[2].executionId)}.zip`,archiveStatus:'not_found'}]};
+ return {root,input,selection,disposition,context,output:path.join(root,'report')};
+}
+async function prepare(f:any){return (aggregate.prepareRetainedCatalogReport ?? aggregate.mergeCatalogProjections)(f.input,f.output,f.selection,f.context,f.disposition,{kind:'operator',revisionSha:'d'.repeat(40)});}
+test('available original PASS and FAIL publish beside explicit UNKNOWN reporting gap instead of all-or-nothing rejection',async t=>{
+ const f=await fixture(t);await assert.rejects(aggregate.mergeCatalogProjections(f.input,path.join(f.root,'old'),f.selection,f.context),/Incomplete projection set/);
+ const receipt=await prepare(f);
+ assert.deepEqual(receipt.coverage,{selected:3,gradedOriginals:2,originalPassed:1,originalFailed:1,reportingGaps:1});
+ const rows=await Promise.all((await readdir(f.output+'/allure')).filter(n=>n.endsWith('-result.json')).map(async n=>JSON.parse(await readFile(f.output+'/allure/'+n,'utf8'))));
+ assert.deepEqual(rows.map(r=>r.status).sort(),['broken','failed','passed']);
+ const gap=rows.find(r=>r.status==='broken');assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102');
+ const data=JSON.parse(await readFile(f.output+'/allure/'+gap.attachments[0].source,'utf8'));
+ assert.equal(data.originalStatus,null);assert.equal(data.checks,null);assert.equal(data.nativeRunCount,null);assert.equal(data.cleanup,null);
+ assert.equal(data.outcome,'reporting_gap');assert.equal(data.doesNotAttestFeatureOutcome,true);
+ assert.equal(receipt.files.length,9);await assert.rejects(prepare(f));
+});
+test('unexplained missing, duplicate, foreign origin and nonterminal evidence refuse without emitting an Allure payload',async t=>{
+ for(const change of [(f:any)=>f.disposition.gaps=[],(f:any)=>f.disposition.gaps.push(f.disposition.gaps[0]),(f:any)=>f.disposition.origin.workflowSha='f'.repeat(40),(f:any)=>f.disposition.upstream.status='in_progress',(f:any)=>f.disposition.selectionSha256='f'.repeat(64),(f:any)=>f.disposition.gaps[0].archiveStatus='present',(f:any)=>f.disposition.gaps[0].executionId=f.selection.cases[0].executionId]){
+  const f=await fixture(t);change(f);await assert.rejects(prepare(f));await assert.rejects(readdir(f.output+'/allure'));
+ }
+});
+test('gap input cannot inject a grade, private text, model usage or a foreign archive',async t=>{
+ for(const change of [(g:any)=>g.status='passed',(g:any)=>g.error='PRIVATE_PROVIDER_PAYLOAD',(g:any)=>g.nativeRunCount=1,(g:any)=>g.archiveKey=g.archiveKey.replace('gha-123','gha-124')]){
+  const f=await fixture(t);change(f.disposition.gaps[0]);await assert.rejects(prepare(f));
+ }
+});
+test('a corrupt authentic projection cannot be downgraded into a gap or bypass original validation',async t=>{
+ const f=await fixture(t);const dir=path.join(f.input,f.selection.cases[0].key);const file=dir+'/binding.json';const binding=JSON.parse(await readFile(file,'utf8'));binding.targetSha='f'.repeat(40);await writeFile(file,JSON.stringify(binding));await assert.rejects(prepare(f));
+});