From 7813e24256cb4f3c1bb9f0dc7a80318bb39f3082 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 01:15:48 -0500 Subject: [PATCH] Permit one recorded revalidation after qualifying identity invalidation --- tests/runner-e2e/chat-campaign.test.ts | 19 ++++++++++++++++++- tests/runner-e2e/chat-campaign.ts | 26 +++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-campaign.test.ts b/tests/runner-e2e/chat-campaign.test.ts index a5c592fd523..f6fea3cbfd6 100644 --- a/tests/runner-e2e/chat-campaign.test.ts +++ b/tests/runner-e2e/chat-campaign.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { expect, it } from "vitest"; -import { campaignSummary, completeCampaignAttempt, freezeChatCampaign, reserveCampaignAttempt, validateFrozenCampaign, type CampaignAttempt, campaignProductIdentityEligible, type CampaignPublishedIdentity, campaignAttemptProcessed } from "./chat-campaign.js"; +import { campaignRevalidationIdentity, campaignSummary, completeCampaignAttempt, freezeChatCampaign, reserveCampaignAttempt, validateFrozenCampaign, type CampaignAttempt, campaignProductIdentityEligible, type CampaignPublishedIdentity, campaignAttemptProcessed } from "./chat-campaign.js"; const root = new URL("../../", import.meta.url).pathname; const base = "8373d7867963674ff960d2345d0d28086f799bc7"; function attempt(ordinal: 1 | 2, executionId: string): CampaignAttempt { @@ -176,3 +176,20 @@ it("permits a processed unsupported remainder while requiring every unfinished b expect(campaignSummary(campaign, temporary)).toMatchObject({ authoringResolved: 243, unfinished: 1, mergeEligible: false }); } finally { rmSync(temporary, { recursive: true, force: true }); } }); + +it("permits exactly one recorded final revalidation only after an actual selected qualifying identity change", () => { + const scenario = freezeChatCampaign(root, base).scenarios.find(x => x.id === "tasks.create-backlog")!, first = attempt(1, scenario.executionId!); + reserveCampaignAttempt(scenario, first); first.outcome = "passed"; const original = JSON.stringify(first); + const next = attempt(2, scenario.executionId!); next.sourceRevision = "c".repeat(40); + next.passedRevalidation = { reason: "Shared selected closure changed for final baseline", previous: campaignRevalidationIdentity(first), next: campaignRevalidationIdentity(next) }; + scenario.qualifyingDefinitionDigest = next.definitionDigest; scenario.qualifyingGraderDigest = next.graderDigest; + expect(() => reserveCampaignAttempt(scenario, next)).toThrow("actual qualifying identity invalidation"); + next.definitionDigest = "actual-new-selected-closure"; scenario.qualifyingDefinitionDigest = next.definitionDigest; + next.passedRevalidation.next = campaignRevalidationIdentity(next); next.passedRevalidation.previous.definitionDigest = "invented-old"; + expect(() => reserveCampaignAttempt(scenario, next)).toThrow("actual qualifying identity invalidation"); + next.passedRevalidation.previous = campaignRevalidationIdentity(first); next.correction = null; + expect(() => reserveCampaignAttempt(scenario, next)).toThrow("recorded correction"); + next.correction = "Coherent shared closure correction on final B"; reserveCampaignAttempt(scenario, next); + expect(JSON.stringify(first)).toBe(original); next.outcome = "failed"; + expect(() => reserveCampaignAttempt(scenario, attempt(2, scenario.executionId!))).toThrow("budget exhausted"); +}); diff --git a/tests/runner-e2e/chat-campaign.ts b/tests/runner-e2e/chat-campaign.ts index ee82c091f06..4284919ba14 100644 --- a/tests/runner-e2e/chat-campaign.ts +++ b/tests/runner-e2e/chat-campaign.ts @@ -55,6 +55,20 @@ export function campaignProductIdentityEligible(scenario: CampaignScenario, atte if (scenario.id === "instance-operations.update-instance") return !!product.target && product.target.version !== product.source.version && samePublishedIdentity(product.target, product.observed); return !product.target && samePublishedIdentity(product.source, product.observed); } +export interface CampaignRevalidationIdentity { + definitionDigest: string; graderDigest: string; productBaseline: string | null; databaseLane: DatabaseFingerprintIdentity; +} +/** Source checkout revisions alone are not qualifying identity changes. */ +export function campaignRevalidationIdentity(attempt: CampaignAttempt): CampaignRevalidationIdentity { + const product = attempt.productIdentity; + let productBaseline: string | null = null; + if (product?.kind === "source-build" || product?.kind === "source-built-installed") productBaseline = JSON.stringify({ kind: product.kind, source: product.productionSourceRevision }); + if (product?.kind === "published-release") { + const pin = (v: CampaignPublishedIdentity) => ({ version: v.version, treeSha256: v.treeSha256, artifacts: [...v.artifacts].sort((a, b) => a.package.localeCompare(b.package)) }); + productBaseline = JSON.stringify({ kind: product.kind, lane: product.declaredLane, source: pin(product.source), target: product.target ? pin(product.target) : null }); + } + return { definitionDigest: attempt.definitionDigest, graderDigest: attempt.graderDigest, productBaseline, databaseLane: attempt.databaseLane ?? { lane: "embedded-postgres" } }; +} export interface CampaignAttempt { /** Stable declared DB environment, excluding passwords/IDs. Missing only on historical embedded attempts. */ databaseLane?: DatabaseFingerprintIdentity; @@ -74,6 +88,8 @@ export interface CampaignAttempt { environment: string; automaticRetries: 0; correction: string | null; + /** One final-B measurement only when actual qualifying identity was invalidated. */ + passedRevalidation?: { reason: string; previous: CampaignRevalidationIdentity; next: CampaignRevalidationIdentity }; classification: string | null; diagnosisHistory?: Array<{ at: string; classification: string | null; detail: string | null; supersededBy: string }>; observedFailure: string | null; @@ -156,13 +172,21 @@ export function reserveCampaignAttempt(scenario: CampaignScenario, attempt: Camp if (!(scenario.readiness === "implemented" || (scenario.readiness === "partial" && scenario.authoringStatus === "concrete-blocker" && scenario.blockerProbeSources?.length)) || !scenario.executionId) throw new Error("Scenario has no executable implementation"); if (scenario.attempts.some(entry => entry.outcome === "running")) throw new Error("Previous attempt still running"); if (scenario.attempts.length >= 2 || attempt.ordinal !== scenario.attempts.length + 1) throw new Error("Attempt budget exhausted or incorrect ordinal"); - if (scenario.attempts[0]?.outcome === "passed") throw new Error("Passing scenario cannot be rerun"); if (attempt.ordinal === 2 && !attempt.correction?.trim()) throw new Error("Fix cycle requires a recorded correction"); if (attempt.automaticRetries !== 0 || attempt.executionId !== scenario.executionId || attempt.outcome !== "running") throw new Error("Invalid attempt reservation"); if ([attempt.sourceRevision, attempt.workingTreeDigest, attempt.definitionDigest, attempt.suiteDefinitionDigest, attempt.graderDigest, attempt.model, attempt.profile, attempt.environment].some(value => !value.trim())) throw new Error("Missing attempt provenance"); const selectedDatabase = databaseFingerprintIdentity(); if (attempt.databaseLane && JSON.stringify(attempt.databaseLane) !== JSON.stringify(selectedDatabase)) throw new Error("Attempt database lane differs from selected fingerprint environment"); attempt.databaseLane ??= selectedDatabase; + const first = scenario.attempts[0]; + if (first?.outcome === "passed") { + const record = attempt.passedRevalidation, previous = campaignRevalidationIdentity(first), next = campaignRevalidationIdentity(attempt); + const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + const actualInvalidation = previous.definitionDigest !== next.definitionDigest || previous.graderDigest !== next.graderDigest + || !same(previous.databaseLane, next.databaseLane) || previous.productBaseline !== null && next.productBaseline !== null && previous.productBaseline !== next.productBaseline && campaignProductIdentityEligible(scenario, attempt); + if (!record?.reason.trim() || !attempt.correction?.trim() || !actualInvalidation || !same(record.previous, previous) || !same(record.next, next) + || attempt.definitionDigest !== scenario.qualifyingDefinitionDigest || attempt.graderDigest !== scenario.qualifyingGraderDigest) throw new Error("Passing scenario cannot be rerun without recorded actual qualifying identity invalidation"); + } else if (attempt.passedRevalidation) throw new Error("Passed-result revalidation requires an original pass"); scenario.attempts.push(attempt); } export function completeCampaignAttempt(scenario: CampaignScenario, completion: {