From aae4f384e0532150e3b8c6ac19c62eee8ab2844c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 03:36:27 -0500 Subject: [PATCH] fix(evals): resolve only anchored complete cleanup evidence --- tests/runner-e2e/chat-campaign.test.ts | 52 ++++++++++++++++++- tests/runner-e2e/chat-campaign.ts | 70 +++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-campaign.test.ts b/tests/runner-e2e/chat-campaign.test.ts index f6fea3cbfd6..ed45b8afa86 100644 --- a/tests/runner-e2e/chat-campaign.test.ts +++ b/tests/runner-e2e/chat-campaign.test.ts @@ -1,10 +1,60 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { createHash } from "node:crypto"; import { expect, it } from "vitest"; -import { campaignRevalidationIdentity, campaignSummary, completeCampaignAttempt, freezeChatCampaign, reserveCampaignAttempt, validateFrozenCampaign, type CampaignAttempt, campaignProductIdentityEligible, type CampaignPublishedIdentity, campaignAttemptProcessed } from "./chat-campaign.js"; +import { appendCampaignCleanupResolution, campaignCleanupResolved, campaignRevalidationIdentity, campaignSummary, completeCampaignAttempt, freezeChatCampaign, reserveCampaignAttempt, validateFrozenCampaign, type CampaignAttempt, campaignProductIdentityEligible, type CampaignPublishedIdentity, campaignAttemptProcessed } from "./chat-campaign.js"; const root = new URL("../../", import.meta.url).pathname; const base = "8373d7867963674ff960d2345d0d28086f799bc7"; +it("resolves only complete hash-bound later resource absence without rewriting the original failure", () => { + const directory = mkdtempSync(path.join(tmpdir(), "chat-cleanup-resolution-")); + try { + const prefix = "tests/runner-e2e/results/cleanup-fixture/"; + mkdirSync(path.join(directory, prefix), { recursive: true }); + const save = (name: string, value: unknown) => { + const text = JSON.stringify(value); const relative = prefix + name; + writeFileSync(path.join(directory, relative), text); + return { path: relative, sha256: createHash("sha256").update(text).digest("hex") }; + }; + const a = attempt(1, "fixture-execution"); a.outcome = "failed"; a.cleanup = "failed"; a.finishedAt = a.startedAt; + const ownership = save("owned.json", { executionId: a.executionId, ordinal: 1, kind: "container", id: "owned-container", owned: true, nonce: "fixture-nonce" }); + const noResources = (scope: string) => save(`${scope}-not-admitted.json`, { executionId: a.executionId, ordinal: 1, scope, notAdmitted: true, observedAt: a.startedAt }); + const original = save("original.json", { schemaVersion: 1, kind: "campaign-admitted-resource-inventory", executionId: a.executionId, ordinal: 1, recordedAt: a.startedAt, cleanup: "failed", + resources: [{ kind: "container", id: "owned-container", ownershipEvidence: ownership }], notAdmittedScopes: { processes: noResources("processes"), proxy: noResources("proxy") } }); + a.evidencePaths = [original.path]; + const observedAt = new Date(Date.parse(a.startedAt) + 1000).toISOString(); + const absence = save("absence.json", { kind: "container", id: "owned-container", absent: true, ownershipEvidenceSha256: ownership.sha256, observedAt }); + const inventory = save("inventory.json", { executionId: a.executionId, ordinal: 1, complete: true, completeScopes: ["processes", "database", "proxy"], + resources: [{ kind: "container", id: "owned-container", ownershipEvidence: ownership }] }); + const body = { schemaVersion: 1, executionId: a.executionId, ordinal: 1, originalCleanup: "failed", observedAt, verifierSourceRevision: base, + originalEvidence: original, inventory, resources: [{ kind: "container", id: "owned-container", absenceEvidence: absence }] }; + const closed = save("resolution.json", body); const resolution = { at: observedAt, receiptPath: closed.path, receiptSha256: closed.sha256 }; + const before = JSON.stringify(a); + expect(campaignCleanupResolved(a, directory)).toBe(false); + appendCampaignCleanupResolution(a, resolution, directory); + expect(campaignCleanupResolved(a, directory)).toBe(true); + const { cleanupResolutions: _, ...unchanged } = a; + expect(JSON.stringify(unchanged)).toBe(before); + expect(() => appendCampaignCleanupResolution(a, resolution, directory)).toThrow("distinct"); + for (const wrong of [ + { ...body, executionId: "foreign" }, { ...body, resources: [] }, + { ...body, inventory: save("partial.json", { executionId: a.executionId, ordinal: 1, complete: false, resources: [] }) }, + { ...body, inventory: save("empty.json", { executionId: a.executionId, ordinal: 1, complete: true, completeScopes: ["processes", "database", "proxy"], resources: [] }), resources: [] }, + { ...body, resources: [{ kind: "container", id: "foreign", absenceEvidence: absence }] }, + ]) { + const ref = save("wrong.json", wrong); + expect(campaignCleanupResolved({ ...a, cleanupResolutions: [{ at: observedAt, receiptPath: ref.path, receiptSha256: ref.sha256 }] }, directory)).toBe(false); + } + const foreignOwnership = save("foreign-owned.json", { executionId: "foreign", ordinal: 1, kind: "container", id: "owned-container", owned: true }); + const foreignOriginal = save("foreign-original.json", { schemaVersion: 1, kind: "campaign-admitted-resource-inventory", executionId: a.executionId, ordinal: 1, recordedAt: a.startedAt, cleanup: "failed", + resources: [{ kind: "container", id: "owned-container", ownershipEvidence: foreignOwnership }], notAdmittedScopes: { processes: noResources("processes"), proxy: noResources("proxy") } }); + const foreignInventory = save("foreign-inventory.json", { executionId: a.executionId, ordinal: 1, complete: true, completeScopes: ["processes", "database", "proxy"], resources: [{ kind: "container", id: "owned-container", ownershipEvidence: foreignOwnership }] }); + const foreignResolution = save("foreign-resolution.json", { ...body, originalEvidence: foreignOriginal, inventory: foreignInventory }); + expect(campaignCleanupResolved({ ...a, evidencePaths: [foreignOriginal.path], cleanupResolutions: [{ at: observedAt, receiptPath: foreignResolution.path, receiptSha256: foreignResolution.sha256 }] }, directory)).toBe(false); + writeFileSync(path.join(directory, absence.path), JSON.stringify({ absent: false })); + expect(campaignCleanupResolved(a, directory)).toBe(false); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); function attempt(ordinal: 1 | 2, executionId: string): CampaignAttempt { return { productIdentity: { kind: "source-build", productionSourceRevision: base, snapshotSourceRevision: base, nativeBinarySha256: "a".repeat(64), workspaceBuildDigest: "b".repeat(64), receiptPaths: ["tests/runner-e2e/results/fixture/source-receipt.json"], coreCandidate: false }, ordinal, outcome: "running", executionId, startedAt: new Date().toISOString(), finishedAt: null, sourceRevision: base, workingTreeDigest: "fixture", definitionDigest: "fixture", suiteDefinitionDigest: "fixture", graderDigest: "fixture", diff --git a/tests/runner-e2e/chat-campaign.ts b/tests/runner-e2e/chat-campaign.ts index 4284919ba14..f32ba5e3120 100644 --- a/tests/runner-e2e/chat-campaign.ts +++ b/tests/runner-e2e/chat-campaign.ts @@ -1,6 +1,6 @@ import { databaseFingerprintIdentity, databaseIdentityMatches, type DatabaseFingerprintIdentity } from "./docker-postgres.js"; import { createHash } from "node:crypto"; -import { readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { readFileSync, readdirSync, writeFileSync, lstatSync, realpathSync } from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { parseDocument } from "yaml"; @@ -96,6 +96,8 @@ export interface CampaignAttempt { evidencePaths: string[]; usage: { costUsd: number | null; coverage: string; tokens: number | null }; cleanup: "pending" | "passed" | "failed" | "not_started"; + /** Later resource verification never rewrites the measured cleanup outcome. */ + cleanupResolutions?: Array<{ at: string; receiptPath: string; receiptSha256: string }>; corePrDependencies: string[]; } export interface CampaignScenario { @@ -231,6 +233,69 @@ export function campaignAttemptProcessed(attempt: CampaignAttempt, root = proces } catch { return false; } }); } + +/** Closed, hash-bound evidence of later absence; an acknowledgement or partial PID census is insufficient. */ +export function campaignCleanupResolved(attempt: CampaignAttempt, root = process.cwd()): boolean { + if (attempt.cleanup === "passed" || attempt.cleanup === "not_started") return true; + if (attempt.outcome === "running" || !attempt.finishedAt) return false; + const retained = (relative: string, sha: string) => { + if (!identityHash(sha) || !retainedIdentityPaths([relative])) throw new Error("Invalid cleanup evidence reference"); + const file = path.resolve(realpathSync(root), relative); + if (!lstatSync(file).isFile() || lstatSync(file).isSymbolicLink() || realpathSync(file) !== file) throw new Error("Cleanup evidence must be a retained regular file"); + const bytes = readFileSync(file); + if (createHash("sha256").update(bytes).digest("hex") !== sha) throw new Error("Cleanup evidence changed"); + return JSON.parse(bytes.toString("utf8")); + }; + return (attempt.cleanupResolutions ?? []).some(resolution => { + try { + const r = retained(resolution.receiptPath, resolution.receiptSha256); + if (r.schemaVersion !== 1 || r.executionId !== attempt.executionId || r.ordinal !== attempt.ordinal || r.originalCleanup !== attempt.cleanup + || r.observedAt !== resolution.at || !identityRevision(r.verifierSourceRevision) || !Number.isFinite(Date.parse(r.observedAt)) + || Date.parse(r.observedAt) < Date.parse(attempt.finishedAt!)) return false; + if (!attempt.evidencePaths.includes(r.originalEvidence.path)) return false; + const original = retained(r.originalEvidence.path, r.originalEvidence.sha256); + if (original.schemaVersion !== 1 || original.kind !== "campaign-admitted-resource-inventory" || original.executionId !== attempt.executionId + || original.ordinal !== attempt.ordinal || original.cleanup !== attempt.cleanup || !Array.isArray(original.resources) + || !Number.isFinite(Date.parse(original.recordedAt)) || Date.parse(original.recordedAt) > Date.parse(attempt.finishedAt!)) return false; + const inventory = retained(r.inventory.path, r.inventory.sha256); + if (inventory.executionId !== attempt.executionId || inventory.ordinal !== attempt.ordinal || inventory.complete !== true + || !["processes", "database", "proxy"].every(scope => inventory.completeScopes?.includes(scope)) || !Array.isArray(inventory.resources)) return false; + const scopeFor = (kind: string) => kind === "process" ? "processes" : kind === "proxy" ? "proxy" : "database"; + for (const scope of ["processes", "database", "proxy"]) { + if (original.resources.some((row: { kind: string }) => scopeFor(row.kind) === scope)) continue; + const unused = original.notAdmittedScopes?.[scope]; + if (!unused) return false; + const evidence = retained(unused.path, unused.sha256); + if (evidence.executionId !== attempt.executionId || evidence.ordinal !== attempt.ordinal || evidence.scope !== scope || evidence.notAdmitted !== true + || !Number.isFinite(Date.parse(evidence.observedAt)) || Date.parse(evidence.observedAt) > Date.parse(attempt.finishedAt!)) return false; + } + const identity = (row: { kind: string; id: string; ownershipEvidence: { path: string; sha256: string } }) => JSON.stringify([row.kind, row.id, row.ownershipEvidence.path, row.ownershipEvidence.sha256]); + if (inventory.resources.length !== original.resources.length + || inventory.resources.some((row: Parameters[0]) => !original.resources.some((saved: Parameters[0]) => identity(row) === identity(saved)))) return false; + if (!Array.isArray(r.resources) || r.resources.length !== inventory.resources.length) return false; + const identities = new Set(); + return inventory.resources.every((resource: { kind: string; id: string; ownershipEvidence: { path: string; sha256: string } }) => { + if (!["process", "container", "volume", "proxy"].includes(resource.kind) || !resource.id) return false; + const key = `${resource.kind}:${resource.id}`; + if (identities.has(key)) return false; + identities.add(key); + const ownership = retained(resource.ownershipEvidence.path, resource.ownershipEvidence.sha256); + if (ownership.executionId !== attempt.executionId || ownership.ordinal !== attempt.ordinal || ownership.kind !== resource.kind + || ownership.id !== resource.id || ownership.owned !== true) return false; + const rows = r.resources.filter((row: { kind: string; id: string }) => row.kind === resource.kind && row.id === resource.id); + if (rows.length !== 1) return false; + const evidence = retained(rows[0].absenceEvidence.path, rows[0].absenceEvidence.sha256); + return evidence.kind === resource.kind && evidence.id === resource.id && evidence.absent === true + && evidence.ownershipEvidenceSha256 === resource.ownershipEvidence.sha256 && evidence.observedAt === r.observedAt; + }); + } catch { return false; } + }); +} +export function appendCampaignCleanupResolution(attempt: CampaignAttempt, resolution: NonNullable[number], root = process.cwd()): void { + if (!["failed", "pending"].includes(attempt.cleanup) || attempt.cleanupResolutions?.some(row => row.receiptPath === resolution.receiptPath)) throw new Error("Cleanup resolution must append distinct evidence to unresolved historical cleanup"); + if (!campaignCleanupResolved({ ...attempt, cleanupResolutions: [resolution] }, root)) throw new Error("Cleanup resolution lacks complete verified resource absence"); + (attempt.cleanupResolutions ??= []).push({ ...resolution }); +} export function campaignSummary(campaign: ChatCampaign, root = process.cwd()) { const initialPassed = campaign.scenarios.filter(scenario => { const first = scenario.attempts[0]; return scenario.readiness === "implemented" && (!scenario.authoringStatus || scenario.authoringStatus === "complete") && first?.outcome === "passed" && first.cleanup === "passed" && first.corePrDependencies.length === 0 && campaignProductIdentityEligible(scenario, first, campaign.baselineRevision) && campaignDatabaseEvidenceEligible(first, root); }).length; const passed = campaign.scenarios.filter(scenario => { const final = scenario.attempts.at(-1); return scenario.readiness === "implemented" && (!scenario.authoringStatus || scenario.authoringStatus === "complete") && final?.outcome === "passed" && final.corePrDependencies.length === 0 && final.cleanup === "passed" && final.definitionDigest === scenario.qualifyingDefinitionDigest && final.graderDigest === scenario.qualifyingGraderDigest && campaignProductIdentityEligible(scenario, final, campaign.baselineRevision) && campaignDatabaseEvidenceEligible(final, root); }).length; @@ -248,7 +313,8 @@ export function campaignSummary(campaign: ChatCampaign, root = process.cwd()) { return { authored, coverageComplete: implemented, concreteBlockers, unfinished, authoringResolved, processed, sourceChatHumanCountScope: "standing-source-chat-only", productEvidence, denominator: campaign.denominator, implemented, attempted, unrun: campaign.denominator - attempted, initialPassed, finalPassed: passed, firstAttemptRate: initialPassed / campaign.denominator, finalRate: passed / campaign.denominator, requiredPasses: campaign.requiredPasses, - mergeEligible: authoringResolved === campaign.denominator && processed === campaign.denominator && passed >= campaign.requiredPasses && !campaign.scenarios.some(scenario => scenario.attempts.some(attempt => attempt.outcome === "running" || attempt.cleanup === "failed" || attempt.cleanup === "pending")) }; + unresolvedCleanup: measured.filter(({ attempt }) => !campaignCleanupResolved(attempt, root)).length, + mergeEligible: authoringResolved === campaign.denominator && processed === campaign.denominator && passed >= campaign.requiredPasses && !campaign.scenarios.some(scenario => scenario.attempts.some(attempt => attempt.outcome === "running" || !campaignCleanupResolved(attempt, root))) }; } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { const [command, baseline] = process.argv.slice(2);