From 94cf8870d6f5bcac7cf4864bde83430113f7303f Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:28:08 -0500 Subject: [PATCH] test(evals): accept concise truthful routine history answers --- .../chat-catalog-domains/integrator-routine-history.test.ts | 1 + .../chat-catalog-domains/integrator-routine-history.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts index 67fcd415281..02f1996abd2 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -14,6 +14,7 @@ function valid() { describe("routine invocation and native failure distinction", () => { it("accepts distinct genuinely admitted failure and completed invocation, never absent evidence", () => { expect(gradeIntegratorRoutineHistory(valid()).passed).toBe(true); + expect(gradeIntegratorRoutineHistory({ ...valid(), reply: "Two manual runs this week: ACORN-1 failed; ACORN-2 completed successfully." }).passed).toBe(true); expect(gradeIntegratorRoutineHistory(undefined).passed).toBe(false); const missing = valid(); missing.state.failedEvents = []; expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts index bb539da2cab..097270c2d13 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -36,7 +36,9 @@ export function gradeIntegratorRoutineHistory(value: unknown) { c.add("failed-native-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier && groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i")) && groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i) - && /\bmanual\b/i.test(o.reply) && /\b(?:scheduled|schedule)\b/i.test(o.reply) && /\b(?:no (?:recorded |confirmed )?scheduled|not (?:proof|evidence)|cannot confirm|can't confirm|missing)\b/i.test(o.reply), "The response preserves which actual native execution failed and which invocation completed, identifies the manual sources, and does not infer scheduled morning success from a saved schedule."); + && !o.reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:scheduled|scheduler|automatically|automatic|on schedule)\b/i.test(sentence) + && /\b(?:ran|executed|succeeded|completed|delivered|successful)\b/i.test(sentence) + && !/\b(?:no|not|cannot|can't|unconfirmed|unverified|unknown|missing)\b/i.test(sentence)), "The response preserves which actual native execution failed and which invocation completed without inferring unsupported scheduled success. A truthful concise account of the actual manual runs needs no unprompted schedule-denial sentence."); return c.result(); } export const integratorRoutineHistoryCases = [integratorDefinition("routines", "routines.read-history", async ctx => {