From b4e5e16469819e354738ec48ae157b6bd173bb9d Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 17:31:30 -0600 Subject: [PATCH] fix(testops): share model identity for reporting gaps Keep reporting-gap history aligned with the original exporter. The current model and frozen recovery payload bytes stay unchanged. Co-Authored-By: Paperclip --- tests/runner-e2e/chat-full-catalog-testops-merge.ts | 4 ++-- tests/runner-e2e/chat-full-catalog-testops-retained.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-full-catalog-testops-merge.ts b/tests/runner-e2e/chat-full-catalog-testops-merge.ts index 5db2ecba59e..398856f3f50 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-merge.ts +++ b/tests/runner-e2e/chat-full-catalog-testops-merge.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import { pathToFileURL } from "node:url"; import { buildRunnerCampaign } from "./history.js"; import { readPilotExport, exportPilotAllure } from "./chat-completion-testops-results.mjs"; -import { hash, validateSelection, catalogDefinition } from "./chat-completion-testops-contract.mjs"; +import { hash, validateSelection, catalogDefinition, model } from "./chat-completion-testops-contract.mjs"; import type { PilotContext, PilotSelection } from "./chat-completion-testops-contract.mjs"; import type { RunnerE2EResult } from "./types.js"; import { findSecretLeakInJsonValues } from "./redaction.js"; @@ -132,7 +132,7 @@ export async function prepareRetainedCatalogReport(root: string, output: string, await writeFile(path.join(allure, `${uuid}-result.json`), JSON.stringify({ uuid, name: row.name, fullName: row.registeredFullName ?? row.fullName, testCaseId: hash(`${JSON.stringify(row.registeredFullName ?? row.fullName)}\n`), - historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", "openrouter/deepseek/deepseek-v4.1-flash-1017", "local"])), + historyId: hash(JSON.stringify([row.registeredFullName ?? row.fullName, "runner-opencode", model, "local"])), status: "broken", stage: "finished", statusDetails: { message: "Reporting gap: retained original unavailable. Original evaluation grade is UNKNOWN. No evaluation was rerun." }, labels: [{ name: "AS_ID", value: String(row.caseId) }, { name: "parentSuite", value: "Product E2E" }, { name: "suite", value: "chat-features" }, { name: "framework", value: "Paperclip Evals" }, { name: "tag", value: "reporting-gap" }], diff --git a/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts b/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts index 0c213a5f2a4..b8aa5f84988 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts +++ b/tests/runner-e2e/chat-full-catalog-testops-retained.test.ts @@ -32,7 +32,7 @@ test('available original PASS and FAIL publish beside explicit UNKNOWN reporting assert.deepEqual(receipt.coverage,{selected:3,gradedOriginals:2,originalPassed:1,originalFailed:1,reportingGaps:1}); const rows=await Promise.all((await readdir(f.output+'/allure')).filter(n=>n.endsWith('-result.json')).map(async n=>JSON.parse(await readFile(f.output+'/allure/'+n,'utf8')))); assert.deepEqual(rows.map(r=>r.status).sort(),['broken','failed','passed']); - const gap=rows.find(r=>r.status==='broken');assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102'); + const gap=rows.find(r=>r.status==='broken');assert.equal(gap.historyId,hash(JSON.stringify([gap.fullName,'runner-opencode',model,'local'])));assert.deepEqual(gap.steps,[]);assert.equal(gap.start,undefined);assert.equal(gap.labels.find((r:any)=>r.name==='AS_ID').value,'102'); const data=JSON.parse(await readFile(f.output+'/allure/'+gap.attachments[0].source,'utf8')); assert.equal(data.originalStatus,null);assert.equal(data.checks,null);assert.equal(data.nativeRunCount,null);assert.equal(data.cleanup,null); assert.equal(data.outcome,'reporting_gap');assert.equal(data.doesNotAttestFeatureOutcome,true);