test(evals): classify settled oracle check failures as candidate failures

"Settled scenario failed: <check ids>" was matched against infrastructure
words, so the check id specific-operator-qualification-next-step made a grader
rejection permanent_infrastructure (and ids naming a provider timeout would be
retried as transient). A settled grade is always the candidate's outcome.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/failure-classifier.ts

@@ -28,6 +28,10 @@ export function classifyFailure(error: unknown): FailureClass {
     return "transient_infrastructure";
   if (/secret.*(?:leak|plaintext|redaction)/i.test(message))
     return "secret_leak";
+  // A settled chat-catalog grade lists the oracle's failed check ids. Those ids are
+  // free-form names ("specific-operator-qualification-next-step") and must not be
+  // read as infrastructure words: the oracle rejected the candidate's outcome.
+  if (/\bSettled scenario failed:/.test(message)) return "candidate_failure";
   if (/cleanup|teardown|lease.*release/i.test(message)) {
     return TRANSIENT.test(message)
       ? "transient_infrastructure"

tests/runner-e2e/support.test.ts

@@ -803,6 +803,12 @@ describe("runner E2E run observations", () => {
 });
 
 describe("runner E2E failure policy", () => {
+  it("classifies settled oracle check failures as candidate failures whatever the check ids say", () => {
+    for (const ids of ["specific-operator-qualification-next-step", "provider-connection-timeout-explained", "unauthorized-write-denied, cleanup-reported"])
+      expect(classifyFailure(new Error(`Stopped waiting for agents.managed-profile: Settled scenario failed: ${ids}`))).toBe("candidate_failure");
+    expect(classifyFailure(new Error("Stopped waiting for x: Settled scenario failed: a; secret plaintext leak in server.log"))).toBe("secret_leak");
+  });
+
   it("classifies sandbox file-transfer RPC deadlines without hiding other RPC defects", () => {
     for (const method of ["environmentSyncIn", "environmentSyncOut"]) {
       expect(classifyFailure(new Error(