diff --git a/tests/runner-e2e/failure-classifier.ts b/tests/runner-e2e/failure-classifier.ts index 6d2a152fd37..d24dcbff027 100644 --- a/tests/runner-e2e/failure-classifier.ts +++ b/tests/runner-e2e/failure-classifier.ts @@ -28,6 +28,10 @@ export function classifyFailure(error: unknown): FailureClass { return "transient_infrastructure"; if (/secret.*(?:leak|plaintext|redaction)/i.test(message)) return "secret_leak"; + // A settled chat-catalog grade lists the oracle's failed check ids. Those ids are + // free-form names ("specific-operator-qualification-next-step") and must not be + // read as infrastructure words: the oracle rejected the candidate's outcome. + if (/\bSettled scenario failed:/.test(message)) return "candidate_failure"; if (/cleanup|teardown|lease.*release/i.test(message)) { return TRANSIENT.test(message) ? "transient_infrastructure" diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index 37eda47e360..76d8b478ce0 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -803,6 +803,12 @@ describe("runner E2E run observations", () => { }); describe("runner E2E failure policy", () => { + it("classifies settled oracle check failures as candidate failures whatever the check ids say", () => { + for (const ids of ["specific-operator-qualification-next-step", "provider-connection-timeout-explained", "unauthorized-write-denied, cleanup-reported"]) + expect(classifyFailure(new Error(`Stopped waiting for agents.managed-profile: Settled scenario failed: ${ids}`))).toBe("candidate_failure"); + expect(classifyFailure(new Error("Stopped waiting for x: Settled scenario failed: a; secret plaintext leak in server.log"))).toBe("secret_leak"); + }); + it("classifies sandbox file-transfer RPC deadlines without hiding other RPC defects", () => { for (const method of ["environmentSyncIn", "environmentSyncOut"]) { expect(classifyFailure(new Error(