diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 65cfaa34ea0..d8d3a42d750 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3371,3 +3371,46 @@ It uses `local_trusted` authentication and no model provider. Its result does no qualify browser reconnects, active-agent continuity, distinct packaged releases or Railway. Keep failed receipts alongside successful attempts; the private server logs and config files may contain credentials and are not shareable evidence. + +### Report a terminal catalog run from retained evidence + +A missing cell projection makes the normal strict aggregate fail. It must not +silently discard authentic sibling results or turn unavailable evidence into an +original failed evaluation. For an operator recovery, authenticate the original +selection, terminal GitHub jobs and exact retained archive keys first. Recover +available originals through the existing sealed-inventory reader and exporter. +Do not run agents again or change the original grades. + +The aggregate owner has an explicit offline data preparation mode: + +```sh +node --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/chat-full-catalog-testops-merge.ts retained-reporting \ + \ + +``` + +`RetainedDisposition` in that module defines the closed input contract. It binds +its origin to the selection, its selection digest to the exact two-space JSON +serialization, and its terminal-job and custody observations to SHA-256 digests. +Every missing execution needs an explicit `not_found` observation at its exact +source-derived archive key. This input records an operator's authenticated +observations; the offline command does not grant read or publish authority. +Active jobs, unexplained omissions, duplicates and foreign identities fail. + +The command reuses the unchanged strict original exporter. It emits one frozen +Allure bundle and `reporting-manifest.json`. The coverage count separates graded +originals from reporting gaps. Gaps are separate DTOs with UNKNOWN original grade, +checks, native-run count and cleanup. Their TestOps outcome is `broken`, not an +inferred feature failure or pass. They have no fabricated duration or test steps. +Only the flat `allure/` JSON files are transport files; the manifest stays outside +any upload archive. Private originals, screenshots and traces stay private. + +Review the immutable payload and its file hashes before an authorized upload to +the original jobrun. Persist a once-intent before transmission. After an ambiguous +response, reconcile with manual GET requests or an independently reviewed, +gap-aware read-only check; do not resend or regenerate UUIDs. The existing +`verify-upload` CLI accepts strict original outcomes only. It cannot verify the +separate BROKEN gap shape and must not be used for this recovery bundle. Final +coverage must match all selected case IDs and preserve the separate gap count. +This mode prepares data only. It does not upload, start jobs, reopen launches, +change credentials, or change evaluation and evidence limits. diff --git a/tests/runner-e2e/chat-full-catalog-testops-merge.ts b/tests/runner-e2e/chat-full-catalog-testops-merge.ts index 2aec38507f4..740bfdf711f 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-merge.ts +++ b/tests/runner-e2e/chat-full-catalog-testops-merge.ts @@ -1,12 +1,15 @@ import { lstat, mkdir, readFile, readdir, realpath, writeFile } from "node:fs/promises"; import path from "node:path"; +import { randomUUID } from "node:crypto"; import { pathToFileURL } from "node:url"; import { buildRunnerCampaign } from "./history.js"; -import { readPilotExport } from "./chat-completion-testops-results.mjs"; -import { hash, validateSelection } from "./chat-completion-testops-contract.mjs"; +import { readPilotExport, exportPilotAllure } from "./chat-completion-testops-results.mjs"; +import { hash, validateSelection, catalogDefinition, model } from "./chat-completion-testops-contract.mjs"; import type { PilotContext, PilotSelection } from "./chat-completion-testops-contract.mjs"; import type { RunnerE2EResult } from "./types.js"; import { findSecretLeakInJsonValues } from "./redaction.js"; +import { privateArtifactKey } from "./private-history.js"; +import { manualCellKind } from "./manual-private-retention.js"; export async function mergeCatalogProjections(root: string, output: string, selection: PilotSelection, context: PilotContext) { if (context.catalog !== "full") throw new Error("Full catalog context required"); @@ -49,11 +52,129 @@ export async function mergeCatalogProjections(root: string, output: string, sele return binding; } +export interface RetainedDisposition { + schema: "paperclip.chat-catalog-retained-disposition/v1"; + origin: PilotContext & { projectId: number; planId: number; jobId: number; launchId: number; allureJobRunId: string }; + selectionSha256: string; + upstream: { status: "completed"; jobsSha256: string }; + custodySha256: string; + gaps: Array<{ executionId: string; archiveKey: string; archiveStatus: "not_found" }>; +} + +/** Report retained measurements without turning missing evidence into a measurement. + * This is an offline data transform, not dispatch authority. The operator must + * authenticate the selection, terminal jobs and exact-key custody observations + * before invoking it. Origin and recovery executor identities remain separate. + */ +export async function prepareRetainedCatalogReport(root: string, output: string, selection: PilotSelection, context: PilotContext, + disposition: RetainedDisposition, executor: { kind: "operator"; revisionSha: string }) { + const selected = validateSelection(selection, context, selection.allureJobRunId); + const keys = (value: object, expected: string[]) => Object.keys(value).sort().join() === [...expected].sort().join(); + const sha = (value: string) => /^[a-f0-9]{64}$/.test(value); + const origin = { catalog: context.catalog, runId: context.runId, attempt: context.attempt, workflowSha: context.workflowSha, + targetSha: context.targetSha, projectId: selection.projectId, planId: selection.planId, jobId: selection.jobId, + launchId: selection.launchId, allureJobRunId: selection.allureJobRunId }; + if (context.catalog !== "full" || context.targetSha !== catalogDefinition(context).targetSha || context.attempt !== "1" + || !/^[1-9][0-9]{0,15}$/.test(context.runId) || !/^[a-f0-9]{40}$/.test(context.workflowSha) + || !keys(disposition, ["schema", "origin", "selectionSha256", "upstream", "custodySha256", "gaps"]) + || disposition.schema !== "paperclip.chat-catalog-retained-disposition/v1" + || !keys(disposition.origin, Object.keys(origin)) || Object.entries(origin).some(([key, value]) => disposition.origin[key as keyof typeof origin] !== value) + || disposition.selectionSha256 !== hash(JSON.stringify(selection, null, 2)) + || !keys(disposition.upstream, ["status", "jobsSha256"]) || disposition.upstream.status !== "completed" || !sha(disposition.upstream.jobsSha256) + || !sha(disposition.custodySha256) || !Array.isArray(disposition.gaps) || disposition.gaps.length > selected.length + || !keys(executor, ["kind", "revisionSha"]) || executor.kind !== "operator" || !/^[a-f0-9]{40}$/.test(executor.revisionSha)) throw new Error("Invalid retained reporting disposition"); + const gaps = new Set(); + for (const gap of disposition.gaps) { + if (!keys(gap, ["executionId", "archiveKey", "archiveStatus"]) || gaps.has(gap.executionId) + || !selected.some(row => row.executionId === gap.executionId) || gap.archiveStatus !== "not_found" + || gap.archiveKey !== privateArtifactKey(manualCellKind(gap.executionId), context.runId)) throw new Error("Invalid reporting gap"); + gaps.add(gap.executionId); + } + const stat = await lstat(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(root) !== path.resolve(root)) throw new Error("Invalid projection root"); + const originals = new Set(); + for (const name of await readdir(root)) { + if (!/^[A-Za-z0-9._-]+$/.test(name)) throw new Error("Invalid projection path"); + const directory = path.join(root, name), info = await lstat(directory), bindingFile = path.join(directory, "binding.json"); + if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Invalid projection path"); + const bindingStat = await lstat(bindingFile); + if (!bindingStat.isFile() || bindingStat.isSymbolicLink() || bindingStat.nlink !== 1 || bindingStat.size > 262_144) throw new Error("Invalid projection binding"); + const binding = JSON.parse(await readFile(bindingFile, "utf8")); + const id = binding.executionIds?.length === 1 ? binding.executionIds[0] : undefined; + const row = selected.find(value => value.executionId === id); + if (!row || originals.has(id) || gaps.has(id)) throw new Error("Duplicate or foreign retained partition"); + await readPilotExport(directory, context, { ...selection, cases: [row] }); originals.add(id); + } + if (originals.size + gaps.size !== selected.length) throw new Error("Incomplete retained reporting partition"); + await mkdir(output, { mode: 0o700 }); + const originalSelection = { ...selection, cases: selection.cases.filter(row => originals.has(row.executionId)) }; + const allure = path.join(output, "allure"); + let exported: Array<{ caseId: number; executionId: string; status: string; cleanup: string; attachments: unknown[] }> = []; + if (originals.size) { + const merged = path.join(output, "originals"); + await mergeCatalogProjections(root, merged, originalSelection, context); + exported = await exportPilotAllure(merged, allure, context, originalSelection); + } else await mkdir(allure, { mode: 0o700 }); + for (const row of selected.filter(value => gaps.has(value.executionId))) { + const dto = { schema: "paperclip.chat-catalog-reporting-gap/v1", outcome: "reporting_gap", originalStatus: null, + nativeRunCount: null, checks: null, cleanup: null, doesNotAttestFeatureOutcome: true, + executionId: row.executionId, caseId: row.caseId, fullName: row.registeredFullName ?? row.fullName, + origin, recoveryExecutor: executor, reason: "retained_archive_unavailable", + selectionSha256: disposition.selectionSha256, terminalJobsSha256: disposition.upstream.jobsSha256, custodySha256: disposition.custodySha256 }; + const attachments = []; + for (const [name, value] of [["Reporting gap (original grade UNKNOWN)", dto], + ["Retained evidence custody", { schema: "paperclip.chat-catalog-reporting-gap-custody/v1", executionId: row.executionId, + archiveStatus: "not_found_at_expected_retained_key", originalEvidenceAvailable: false, doesNotAttestFeatureOutcome: true, + custodySha256: disposition.custodySha256 }]] as const) { + const source = `${randomUUID()}-attachment.json`; + await writeFile(path.join(allure, source), JSON.stringify(value), { flag: "wx", mode: 0o600 }); + attachments.push({ name, source, type: "application/json" }); + } + const uuid = randomUUID(); + await writeFile(path.join(allure, `${uuid}-result.json`), JSON.stringify({ uuid, name: row.name, + fullName: row.registeredFullName ?? row.fullName, + testCaseId: hash(`${JSON.stringify(row.registeredFullName ?? row.fullName)}\n`), + historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", model, "local"])), + status: "broken", stage: "finished", statusDetails: { message: "Reporting gap: retained original unavailable. Original evaluation grade is UNKNOWN. No evaluation was rerun." }, + labels: [{ name: "AS_ID", value: String(row.caseId) }, { name: "parentSuite", value: "Product E2E" }, + { name: "suite", value: "chat-features" }, { name: "framework", value: "Paperclip Evals" }, { name: "tag", value: "reporting-gap" }], + parameters: [{ name: "canonicalScenario", value: row.key }, { name: "originalGrade", value: "UNKNOWN" }], + links: [{ name: "Original GitHub Actions run", type: "custom", url: `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` }], + steps: [], attachments }), { flag: "wx", mode: 0o600 }); + } + const files = []; + for (const name of (await readdir(allure)).sort()) { + const bytes = await readFile(path.join(allure, name)); + if (findSecretLeakInJsonValues(JSON.parse(bytes.toString()), [], { includeShapes: true })) throw new Error("Unsafe reporting payload"); + files.push({ path: name, bytes: bytes.length, sha256: hash(bytes) }); + } + if (files.length !== selected.length * 3 || files.reduce((sum, row) => sum + row.bytes, 0) > 32 * 1024 * 1024) throw new Error("Invalid reporting payload inventory"); + const receipt = { schema: "paperclip.chat-catalog-retained-report/v1", origin, recoveryExecutor: executor, + selectionSha256: disposition.selectionSha256, dispositionSha256: hash(JSON.stringify(disposition)), + coverage: { selected: selected.length, gradedOriginals: originals.size, + originalPassed: exported.filter(row => row.status === "passed").length, originalFailed: exported.filter(row => row.status !== "passed").length, + reportingGaps: gaps.size }, + rows: selected.map(row => ({ executionId: row.executionId, caseId: row.caseId, + originalStatus: exported.find(value => value.executionId === row.executionId)?.status ?? null, + reportingStatus: exported.find(value => value.executionId === row.executionId)?.status ?? "broken" })), files }; + await writeFile(path.join(output, "reporting-manifest.json"), JSON.stringify(receipt, null, 2), { flag: "wx", mode: 0o600 }); + return receipt; +} + if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.argv[2] === "retained-reporting") { + const [root, output, selectionFile, dispositionFile, executorSha] = process.argv.slice(3); + const selection = JSON.parse(await readFile(selectionFile!, "utf8")); + const context = { catalog: "full" as const, runId: selection.runId, attempt: selection.attempt, + workflowSha: selection.workflowSha, targetSha: selection.targetSha }; + await prepareRetainedCatalogReport(path.resolve(root!), path.resolve(output!), selection, context, + JSON.parse(await readFile(dispositionFile!, "utf8")), { kind: "operator", revisionSha: executorSha! }); + } else { const bytes = await readFile(process.env.TESTOPS_CATALOG_SELECTION!); if (hash(bytes) !== process.env.EXPECTED_SELECTION_SHA256) throw new Error("Selection digest mismatch"); const selection = JSON.parse(bytes.toString("utf8")); const context = { catalog: "full" as const, runId: process.env.GITHUB_RUN_ID!, attempt: process.env.GITHUB_RUN_ATTEMPT!, workflowSha: process.env.GITHUB_SHA!, targetSha: selection.targetSha }; await mergeCatalogProjections(path.resolve("catalog-projections"), path.resolve("runner-e2e-testops"), selection, context); + } } diff --git a/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts b/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts new file mode 100644 index 00000000000..720ca985c0f --- /dev/null +++ b/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts @@ -0,0 +1,55 @@ +import { it as test, afterEach } from 'vitest'; +import assert from 'node:assert/strict'; +import {mkdtemp,mkdir,readFile,readdir,writeFile,rm,realpath} from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; +import * as aggregate from './chat-full-catalog-testops-merge.js'; +import {hash,model} from './chat-completion-testops-contract.mjs'; +import {fullCatalogCases,catalogTargetSha} from './chat-full-catalog-testops-manifest.mjs'; +import {privateArtifactKey} from './private-history.js'; +import {manualCellKind} from './manual-private-retention.js'; +const roots:string[]=[]; +afterEach(async()=>{await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); +const context={catalog:'full' as const,runId:'123',attempt:'1',workflowSha:'a'.repeat(40),targetSha:catalogTargetSha}; +async function fixture(t:any){ + const root=await realpath(await mkdtemp(path.join(os.tmpdir(),'chat-retained-')));roots.push(root); + const input=path.join(root,'inputs');await mkdir(input); + const cases=fullCatalogCases.slice(0,3).map((row,i)=>({...row,caseId:100+i})); + const selection={schema:'paperclip.chat-completion-testops-selection/v1',...context,projectId:1,planId:102,jobId:100,launchId:12,allureJobRunId:'77',cases}; + for(const [index,row] of cases.slice(0,2).entries()){ + const dir=path.join(input,row.key);await mkdir(dir); + const passed=index===0; + const result={schema:'paperclip.runner-e2e.result/v2',suiteDefinitionHash:'c'.repeat(64),startedAt:'2026-11-01T00:00:00Z',finishedAt:'2026-11-01T00:00:01Z',durationMs:1000,executionId:row.executionId,suiteId:'chat-features',caseId:row.scenario,profileId:'runner-opencode',provider:'opencode',environmentId:'local',model,runtimeMode:'native',attempt:1,source:{sha:catalogTargetSha,ref:null,workflowRunUrl:'https://github.com/paperclipai/paperclip/actions/runs/123'},status:passed?'passed':'failed',cleanup:passed?'passed':'failed',runIds:passed?Array.from({length:row.minimumRuns},(_,i)=>`owned-${i}`):[],matcherResults:passed?Array.from({length:row.checks??2},(_,i)=>({matcher:{kind:'json_path',path:`fixture.check${i}`,expected:true},passed:true,detail:'Retained independent check'})):[],screenshots:[]}; + const files=new Map([['normalized-results.json',Buffer.from(JSON.stringify({schema:'paperclip.runner-e2e.campaign/v2',campaignId:'gha-123-1',retries:0,results:[result]}))],['junit.xml',Buffer.from('')]]); + for(const [name,data] of files)await writeFile(path.join(dir,name),data); + await writeFile(path.join(dir,'binding.json'),JSON.stringify({schema:'paperclip.chat-completion-testops-export/v1',...context,campaignId:'gha-123-1',executionIds:[row.executionId],files:[...files].map(([name,data])=>({path:name,bytes:data.length,sha256:hash(data)})),privateEvidence:[{executionId:row.executionId,requiredEvidenceComplete:passed,finalScreenshot:passed?{sha256:'e'.repeat(64),bytes:24}:null}]})); + } + const disposition={schema:'paperclip.chat-catalog-retained-disposition/v1',origin:{...context,projectId:1,planId:102,jobId:100,launchId:12,allureJobRunId:'77'},selectionSha256:hash(JSON.stringify(selection,null,2)),upstream:{status:'completed',jobsSha256:'b'.repeat(64)},custodySha256:'c'.repeat(64),gaps:[{executionId:cases[2].executionId,archiveKey:privateArtifactKey(manualCellKind(cases[2].executionId),context.runId),archiveStatus:'not_found'}]}; + return {root,input,selection,disposition,context,output:path.join(root,'report')}; +} +async function prepare(f:any){return (aggregate.prepareRetainedCatalogReport ?? aggregate.mergeCatalogProjections)(f.input,f.output,f.selection,f.context,f.disposition,{kind:'operator',revisionSha:'d'.repeat(40)});} +test('available original PASS and FAIL publish beside explicit UNKNOWN reporting gap instead of all-or-nothing rejection',async t=>{ + const f=await fixture(t);await assert.rejects(aggregate.mergeCatalogProjections(f.input,path.join(f.root,'old'),f.selection,f.context),/Incomplete projection set/); + const receipt=await prepare(f); + assert.deepEqual(receipt.coverage,{selected:3,gradedOriginals:2,originalPassed:1,originalFailed:1,reportingGaps:1}); + const rows=await Promise.all((await readdir(f.output+'/allure')).filter(n=>n.endsWith('-result.json')).map(async n=>JSON.parse(await readFile(f.output+'/allure/'+n,'utf8')))); + assert.deepEqual(rows.map(r=>r.status).sort(),['broken','failed','passed']); + const gap=rows.find(r=>r.status==='broken');assert.equal(gap.historyId,hash(JSON.stringify([gap.fullName,'runner-opencode',model,'local'])));assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102'); + const data=JSON.parse(await readFile(f.output+'/allure/'+gap.attachments[0].source,'utf8')); + assert.equal(data.originalStatus,null);assert.equal(data.checks,null);assert.equal(data.nativeRunCount,null);assert.equal(data.cleanup,null); + assert.equal(data.outcome,'reporting_gap');assert.equal(data.doesNotAttestFeatureOutcome,true); + assert.equal(receipt.files.length,9);await assert.rejects(prepare(f)); +}); +test('unexplained missing, duplicate, foreign origin and nonterminal evidence refuse without emitting an Allure payload',async t=>{ + for(const change of [(f:any)=>f.disposition.gaps=[],(f:any)=>f.disposition.gaps.push(f.disposition.gaps[0]),(f:any)=>f.disposition.origin.workflowSha='f'.repeat(40),(f:any)=>f.disposition.upstream.status='in_progress',(f:any)=>f.disposition.selectionSha256='f'.repeat(64),(f:any)=>f.disposition.gaps[0].archiveStatus='present',(f:any)=>f.disposition.gaps[0].executionId=f.selection.cases[0].executionId]){ + const f=await fixture(t);change(f);await assert.rejects(prepare(f));await assert.rejects(readdir(f.output+'/allure')); + } +}); +test('gap input cannot inject a grade, private text, model usage or a foreign archive',async t=>{ + for(const change of [(g:any)=>g.status='passed',(g:any)=>g.error='PRIVATE_PROVIDER_PAYLOAD',(g:any)=>g.nativeRunCount=1,(g:any)=>g.archiveKey=g.archiveKey.replace('gha-123','gha-124')]){ + const f=await fixture(t);change(f.disposition.gaps[0]);await assert.rejects(prepare(f)); + } +}); +test('a corrupt authentic projection cannot be downgraded into a gap or bypass original validation',async t=>{ + const f=await fixture(t);const dir=path.join(f.input,f.selection.cases[0].key);const file=dir+'/binding.json';const binding=JSON.parse(await readFile(file,'utf8'));binding.targetSha='f'.repeat(40);await writeFile(file,JSON.stringify(binding));await assert.rejects(prepare(f)); +});