Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-03 17:51:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(testops): share model identity for reporting gaps

Browse Source 
Keep reporting-gap history aligned with the original exporter. The current model and frozen recovery payload bytes stay unchanged.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 17:31:30 -06:00
1 parent 2704ce6b3c
commit b4e5e16469
2 files changed
+3 -3
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-full-catalog-testops-merge.ts
+2 -2
View File
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import { pathToFileURL } from "node:url";
import { buildRunnerCampaign } from "./history.js";
import { readPilotExport, exportPilotAllure } from "./chat-completion-testops-results.mjs";
import { hash, validateSelection, catalogDefinition } from "./chat-completion-testops-contract.mjs";
import { hash, validateSelection, catalogDefinition, model } from "./chat-completion-testops-contract.mjs";
import type { PilotContext, PilotSelection } from "./chat-completion-testops-contract.mjs";
import type { RunnerE2EResult } from "./types.js";
import { findSecretLeakInJsonValues } from "./redaction.js";
@@ -132,7 +132,7 @@ export async function prepareRetainedCatalogReport(root: string, output: string,
await writeFile(path.join(allure, `${uuid}-result.json`), JSON.stringify({ uuid, name: row.name,
fullName: row.registeredFullName ?? row.fullName,
testCaseId: hash(`${JSON.stringify(row.registeredFullName ?? row.fullName)}\n`),
historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", "openrouter/deepseek/deepseek-v4.1-flash-1017", "local"])),
historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", model, "local"])),
status: "broken", stage: "finished", statusDetails: { message: "Reporting gap: retained original unavailable. Original evaluation grade is UNKNOWN. No evaluation was rerun." },
labels: [{ name: "AS_ID", value: String(row.caseId) }, { name: "parentSuite", value: "Product E2E" },
{ name: "suite", value: "chat-features" }, { name: "framework", value: "Paperclip Evals" }, { name: "tag", value: "reporting-gap" }],
tests/runner-e2e/chat-full-catalog-testops-retained.test.ts
+1 -1
View File
@@ -32,7 +32,7 @@ test('available original PASS and FAIL publish beside explicit UNKNOWN reporting
assert.deepEqual(receipt.coverage,{selected:3,gradedOriginals:2,originalPassed:1,originalFailed:1,reportingGaps:1});
const rows=await Promise.all((await readdir(f.output+'/allure')).filter(n=>n.endsWith('-result.json')).map(async n=>JSON.parse(await readFile(f.output+'/allure/'+n,'utf8'))));
assert.deepEqual(rows.map(r=>r.status).sort(),['broken','failed','passed']);
const gap=rows.find(r=>r.status==='broken');assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102');
const gap=rows.find(r=>r.status==='broken');assert.equal(gap.historyId,hash(JSON.stringify([gap.fullName,'runner-opencode',model,'local'])));assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102');
const data=JSON.parse(await readFile(f.output+'/allure/'+gap.attachments[0].source,'utf8'));
assert.equal(data.originalStatus,null);assert.equal(data.checks,null);assert.equal(data.nativeRunCount,null);assert.equal(data.cleanup,null);
assert.equal(data.outcome,'reporting_gap');assert.equal(data.doesNotAttestFeatureOutcome,true);