From 57b559cde7e9e06997092fa28a08a45cc0728e72 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 14:06:46 -0600 Subject: [PATCH] test(evals): accept production cancellation intent ids; name each post-seal step The native Stop boundary required a bare UUID intent id, but production writes native-cancellation:. Label the health-detail and source-SDK attach steps and code the remaining diagnostic and SDK-proof refusals. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-native-stop-boundary.test.ts | 2 +- tests/runner-e2e/chat-native-stop-boundary.ts | 4 +++- tests/runner-e2e/launch.ts | 2 ++ tests/runner-e2e/private-attempt-diagnostic.test.ts | 2 +- tests/runner-e2e/private-attempt-diagnostic.ts | 11 ++++++++++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-native-stop-boundary.test.ts b/tests/runner-e2e/chat-native-stop-boundary.test.ts index 8d0570fdba7..9002377f8ed 100644 --- a/tests/runner-e2e/chat-native-stop-boundary.test.ts +++ b/tests/runner-e2e/chat-native-stop-boundary.test.ts @@ -2,7 +2,7 @@ import { expect, it, vi } from "vitest"; import type { ChatCatalogCase, ChatCatalogContext, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js"; import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; import { assertNativeStopCallback, nativeStopPendingReceiptMatches, prepareNativeStopCanonicalBoundary, validNativeStopDeclaration } from "./chat-native-stop-boundary.js"; -const receipt: NativeStopBoundaryReceipt = { runId: "11111111-1111-4111-8111-111111111111", issueId: "22222222-2222-4222-8222-222222222222", intentId: "33333333-3333-4333-8333-333333333333", intentRecordedAt: "2026-10-26T12:00:00.000Z" }; +const receipt: NativeStopBoundaryReceipt = { runId: "11111111-1111-4111-8111-111111111111", issueId: "22222222-2222-4222-8222-222222222222", intentId: "native-cancellation:33333333-3333-4333-8333-333333333333", intentRecordedAt: "2026-10-26T12:00:00.000Z" }; const metadata: ChatCatalogDescriptor = { scenarioId: "recovery.post-stop-message", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", sourceFiles: ["runtime-post-stop.ts"], modulePath: "./chat-catalog-domains/runtime-post-stop.js", exportName: "cases", oracleVersion: "control", expectedRunCount: 3, expectedHumanMessageCount: 2, setupHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 600000 }; function fixture() { const run = { id: receipt.runId, companyId: "company", nativeIssueId: receipt.issueId, agentId: "lead", runtimeMode: "native", status: "running", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: receipt.runId, issueId: receipt.issueId, scope: "run", intentId: receipt.intentId, recordedAt: receipt.intentRecordedAt, dispatchState: "pending", dispatched: false } } }; diff --git a/tests/runner-e2e/chat-native-stop-boundary.ts b/tests/runner-e2e/chat-native-stop-boundary.ts index 7f612942201..18162978f70 100644 --- a/tests/runner-e2e/chat-native-stop-boundary.ts +++ b/tests/runner-e2e/chat-native-stop-boundary.ts @@ -13,7 +13,9 @@ export function assertNativeStopCallback(implementation: ChatCatalogCase) { // A request receipt or a cancelled status alone does not establish this interval. export function nativeStopPendingReceiptMatches(input: { receipt: NativeStopBoundaryReceipt; run: { id?: string; companyId?: string; nativeIssueId?: string; agentId?: string; runtimeMode?: string; status?: string; resultJson?: { nativeCancellation?: Record } }; companyId: string; sourceIssueId: string; assignedAgentId: string; setupRunIds: string[] }) { const { receipt: r, run } = input, intent = run.resultJson?.nativeCancellation; - return !!r && Object.keys(r).every(key => ["runId", "issueId", "intentId", "intentRecordedAt"].includes(key)) && [r.runId, r.issueId, r.intentId].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) + return !!r && Object.keys(r).every(key => ["runId", "issueId", "intentId", "intentRecordedAt"].includes(key)) && [r.runId, r.issueId].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) + // Production intent ids are `native-cancellation:` (native-cancellation-request.ts). + && typeof r.intentId === "string" && /^native-cancellation:[a-f0-9-]{36}$/i.test(r.intentId) && Number.isFinite(Date.parse(r.intentRecordedAt)) && input.setupRunIds.includes(r.runId) && run.id === r.runId && run.companyId === input.companyId && run.nativeIssueId === r.issueId && r.issueId === input.sourceIssueId && run.agentId === input.assignedAgentId && run.runtimeMode === "native" && run.status === "running" && intent?.schema === "paperclip.native-cancellation.v1" && intent.companyId === input.companyId && intent.runId === r.runId && intent.issueId === r.issueId && intent.intentId === r.intentId && intent.recordedAt === r.intentRecordedAt && intent.scope === "run" && intent.dispatchState === "pending" && intent.dispatched === false && intent.acknowledgedAt == null; diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index ad27bfe0052..df8c40e32ca 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -950,7 +950,9 @@ async function runAttempt(input: { await writeFile(path.join(uploadDir, "result.json"), safeResult, "utf8"); } } + if (manualDiagnostic) manualDiagnostic.stage = "attach-health-detail"; await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials); + if (manualDiagnostic) manualDiagnostic.stage = "attach-source-sdk"; await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, credentials, repositoryRoot); console.log( `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`, diff --git a/tests/runner-e2e/private-attempt-diagnostic.test.ts b/tests/runner-e2e/private-attempt-diagnostic.test.ts index 71df68dc0a1..b873660aed8 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.test.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.test.ts @@ -123,7 +123,7 @@ it("retains each remaining fixed scanner reason without dynamic text", () => { import { readFileSync } from "node:fs"; it("gives every fixed packaging, sealing and report-asset refusal a closed finalization code", () => { - const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts", "private-framework-report.ts", "redaction.ts", "manual-private-retention.ts", "chat-readiness-detail.ts", "source-sdk-evidence.ts"].flatMap(file => + const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts", "private-framework-report.ts", "redaction.ts", "manual-private-retention.ts", "chat-readiness-detail.ts", "source-sdk-evidence.ts", "private-diagnostics.ts", "source-sdk-public-type.ts"].flatMap(file => [...readFileSync(new URL(`./${file}`, import.meta.url), "utf8").matchAll(/(?:new Error|budgetFailure)\("([^"$`]+)"/g)].map(match => match[1]!)); expect(messages.length).toBeGreaterThan(20); for (const message of messages) { diff --git a/tests/runner-e2e/private-attempt-diagnostic.ts b/tests/runner-e2e/private-attempt-diagnostic.ts index da20a3f3a17..7cf2d9a58f0 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.ts @@ -50,6 +50,15 @@ function carrierReasons(input: unknown, source: "reporter" | "packaging") { })); } const failures = new Map([ + ["Invalid private diagnostic registry path", "invalid-private-diagnostic-registry-path"], + ["Private diagnostic registry ownership or permissions are invalid", "private-diagnostic-registry-ownership-or-permissions"], + ["Invalid private diagnostic registry", "invalid-private-diagnostic-registry"], + ["Private diagnostic registry requires an owned temporary root", "private-diagnostic-registry-requires-an-owned"], + ["Private diagnostic registry execution mismatch", "private-diagnostic-registry-execution-mismatch"], + ["Private diagnostic secret registration must be a string", "private-diagnostic-secret-registration-must-be"], + ["Diagnostic expansion has no raw-root provenance", "diagnostic-expansion-has-no-rawroot-provenance"], + ["SDK public source proof unavailable", "sdk-public-source-proof-unavailable"], + ["SDK public source parse unavailable", "sdk-public-source-parse-unavailable"], ["Bounded diagnostic unavailable", "bounded-diagnostic-unavailable"], ["Diagnostic withheld", "diagnostic-withheld"], ["Diagnostic custody invalid", "diagnostic-custody-invalid"], @@ -191,7 +200,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { return { schema: "paperclip.private-attempt-diagnostic/v1", packagingBudget: closedPackagingBudget(record(input.error).evidenceBudget ?? evidence.budget, message), - stage: member(input.stage, ["execution", "package-original", "seal-original", "attach-original"]), + stage: member(input.stage, ["execution", "package-original", "seal-original", "attach-original", "attach-health-detail", "attach-source-sdk"]), status: member(result.status, ["passed", "failed", "skipped"]), failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]), finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file"