diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 9bbfbec174e..77b3a6285d8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4688,7 +4688,7 @@ "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v2", + "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v3", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts index 9e045a4f010..4be84a6d754 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -126,3 +126,24 @@ describe("complete normal accounting stays mandatory", () => { const foreign = phase(true); foreign.identity.runId = "other"; expect(routineHistoryAccountingReady(foreign.identity, [foreign.snapshot.run!], overview())).toBe(false); }); }); +// Recorded reply from run 42584001490 (prefix renamed); it failed v2 only because "published successfully and is done" missed a keyword. +const RECORDED = "Yes \u2014 the **Morning Launch summary** routine ran twice this week, both today (Nov 9):\n\n- **Failed:** run `51e8bf51` at 08:20 UTC. [ACORN-1](/ACORN/issues/ACORN-1) was blocked because the Synthetic Launch Notes `launch_update` publish returned `Synthetic write failed; page unchanged`. It remains blocked; owner is the Synthetic Launch Notes provider, and the unblock action is restoring `launch_update`.\n- **Completed:** run `4b180bf1` at 08:21 UTC. [ACORN-2](/ACORN/issues/ACORN-2) published successfully and is done.\n\nThe routine has no scheduled triggers, so both were manual runs. The failed run is [run 51e8bf51](/ACORN/agents/59e1d949-9e46-4e57-9f36-64c57d8eb4d5/runs/51e8bf51-94dc-47ca-8f91-2116dbe02979); its agent heartbeat run exited successfully, but the routine run is recorded as failed because its task moved to blocked."; +describe("per-task outcomes are graded against recorded status", () => { + const withReply = (reply: string) => { const v = valid(); v.reply = reply; return gradeIntegratorRoutineHistory(v).passed; }; + it("accepts the recorded reply and rewordings that pair each task with its recorded outcome", () => { + for (const reply of [RECORDED, + "Yes, twice (both manual). [ACORN-1](/ACORN/issues/ACORN-1) is the one that failed: the publish returned an error and the task is blocked. [ACORN-2](/ACORN/issues/ACORN-2) went through and is done.", + "| Task | Outcome |\n|---|---|\n| [ACORN-1](/ACORN/issues/ACORN-1) | Blocked - publish error, nothing was published |\n| [ACORN-2](/ACORN/issues/ACORN-2) | Done, published once |", + "ACORN-1 didn't succeed: the publish failed. ACORN-2 finished and its update was published."]) + expect(withReply(reply), reply).toBe(true); + }); + it("rejects swapped, blanket, mixed and missing per-task outcomes", () => { + for (const reply of [RECORDED.replace("[ACORN-1](/ACORN/issues/ACORN-1)", "[TMP](/ACORN/issues/TMP)").replace(/ACORN-2/g, "ACORN-1").replace(/TMP/g, "ACORN-2"), + "The routine ran twice. ACORN-1 completed fine. ACORN-2 failed to publish.", + "The routine ran twice: ACORN-1 and ACORN-2. Both runs completed successfully.", + "ACORN-1 failed. ACORN-2 failed too, although its run completed.", + "ACORN-1 failed. The other run worked.", + "Runs this week: [ACORN-1](/ACORN/issues/ACORN-1), [ACORN-2](/ACORN/issues/ACORN-2)."]) + expect(withReply(reply), reply).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts index e13e52fa1b2..32e01df8e49 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -6,7 +6,7 @@ import { fixtureWorker } from "./internal-definition.js"; import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js"; import { prepareHistoryNotes, cleanupHistoryNotes, historyNotesPhase, historyNotesCalls, waitHistoryWork, readHistoryWork, genuineHistoryWork, routineHistoryAccountingReady, completeHistoryUsage, type HistoryWorkIdentity, type HistoryWorkSnapshot, type WorkCall } from "./routine-history-work-failure.js"; export { routineHistoryAccountingReady } from "./routine-history-work-failure.js"; -import { groundedStatement, integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; interface HistoryObservation extends InternalObservation { expectedProvider: string; routine: RoutineDetail; executions: RoutineRunSummary[] } const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; async function observeHistory(ctx: ChatCatalogContext, state: Record): Promise { @@ -20,6 +20,34 @@ function historyObservation(value: unknown): HistoryObservation | undefined { return observation(value) && !!o.routine && Array.isArray(o.executions) ? o : undefined; } const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +/** Text the reply attributes to exactly one of the routine's tasks: a line (list item, table row or paragraph) + * that names or links one task, or, where a line names both, each sentence that names one. */ +export function taskOutcomeUnits(reply: string, tasks: Entity[]) { + const names = (task: Entity) => [task.identifier, task.id].filter((value): value is string => !!value); + const mentions = (text: string, task: Entity) => names(task).some(name => new RegExp(`(? [task.id, [] as string[]])); + for (const line of reply.split(/\n+/)) { + const named = tasks.filter(task => mentions(line, task)); + if (named.length === 1) units.get(named[0]!.id)!.push(line); + else if (named.length > 1) for (const sentence of line.split(/(?<=[.!?;])\s+/)) { + const own = tasks.filter(task => mentions(sentence, task)); if (own.length === 1) units.get(own[0]!.id)!.push(sentence); + } + } + return units; +} +const negated = String.raw`(? /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence)); + return failed.some(unit => FAILURE.test(unit)) && completed.some(unit => SUCCESS.test(unit)) + && !failed.some(unit => SUCCESS.test(unit)) && !completed.some(unit => FAILURE.test(unit)) && !blanket; +} export function gradeIntegratorRoutineHistory(value: unknown) { const c = checks(); const o = historyObservation(value); c.add("actual-routine-and-native-execution-history", !!o, "Requires real public invocation history plus distinct actual failed work and successful execution receipts."); @@ -48,8 +76,8 @@ export function gradeIntegratorRoutineHistory(value: unknown) { && o.executions.length === 2 && failedInvocation?.source === "manual" && completedInvocation?.source === "manual", "Both actual manual invocations fall inside the measured current week, with no hidden or duplicate history rows."); c.add("actual-failure-and-completion-linked", !!failedTask && !!completedTask && hasLink(o, "issues", failedTask) && hasLink(o, "issues", completedTask), "The answer links both real execution tasks."); c.add("failed-work-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier - && groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i")) - && groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i) + && failedTask.status === "blocked" && failedInvocation?.status === "failed" && completedTask.status === "done" && completedInvocation?.status === "completed" + && preservesRecordedOutcomes(o.reply, failedTask, completedTask) && !o.reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:scheduled|scheduler|automatically|automatic|on schedule)\b/i.test(sentence) && /\b(?:ran|executed|succeeded|completed|delivered|successful)\b/i.test(sentence) && !/\b(?:no|not|cannot|can't|unconfirmed|unverified|unknown|missing)\b/i.test(sentence)), "The response preserves which actual routine work failed and which invocation completed without inferring unsupported scheduled success. A truthful concise account of the actual manual runs needs no unprompted schedule-denial sentence."); @@ -90,5 +118,5 @@ export const integratorRoutineHistoryCases = [integratorDefinition("routines", " const routine = state.routine as RoutineDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title); }, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/routine-history-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"], - oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v2", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000, + oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v3", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000, })];