diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index d9955a42936..36d708eb7ad 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -697,7 +697,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v3", + "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v4", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -4688,7 +4688,7 @@ "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v3", + "oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v4", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts index 4be84a6d754..775cfcb4129 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -128,10 +128,12 @@ describe("complete normal accounting stays mandatory", () => { }); // Recorded reply from run 42584001490 (prefix renamed); it failed v2 only because "published successfully and is done" missed a keyword. const RECORDED = "Yes \u2014 the **Morning Launch summary** routine ran twice this week, both today (Nov 9):\n\n- **Failed:** run `51e8bf51` at 08:20 UTC. [ACORN-1](/ACORN/issues/ACORN-1) was blocked because the Synthetic Launch Notes `launch_update` publish returned `Synthetic write failed; page unchanged`. It remains blocked; owner is the Synthetic Launch Notes provider, and the unblock action is restoring `launch_update`.\n- **Completed:** run `4b180bf1` at 08:21 UTC. [ACORN-2](/ACORN/issues/ACORN-2) published successfully and is done.\n\nThe routine has no scheduled triggers, so both were manual runs. The failed run is [run 51e8bf51](/ACORN/agents/59e1d949-9e46-4e57-9f36-64c57d8eb4d5/runs/51e8bf51-94dc-47ca-8f91-2116dbe02979); its agent heartbeat run exited successfully, but the routine run is recorded as failed because its task moved to blocked."; +// Second recorded reply (run 42594125035): v3 first misread the quoted tool output `success: false` as a success claim. +const RECORDED_2 = "Yes \u2014 the \"Morning Launch summary\" routine ran this week, twice this morning (Mon, Nov 9).\n\n- **Failed run:** `f0c38bf6\u2026` at 09:41 \u2014 its task moved to blocked because the Synthetic Launch Notes `launch_update` returned `success: false` (\"Synthetic write failed; page unchanged\"). [ACORN-1](/ACORN/issues/ACORN-1) remains blocked.\n- **Succeeded run:** `b050bd96\u2026` at 09:42 \u2014 published successfully; [ACORN-2](/ACORN/issues/ACORN-2) is done.\n\nNo agent heartbeat/execution runs are marked failed \u2014 all three recorded runs show `succeeded`; the failure is recorded at the routine-run level."; describe("per-task outcomes are graded against recorded status", () => { const withReply = (reply: string) => { const v = valid(); v.reply = reply; return gradeIntegratorRoutineHistory(v).passed; }; it("accepts the recorded reply and rewordings that pair each task with its recorded outcome", () => { - for (const reply of [RECORDED, + for (const reply of [RECORDED, RECORDED_2, "`ACORN-1` failed. `ACORN-2` is done.", "Yes, twice (both manual). [ACORN-1](/ACORN/issues/ACORN-1) is the one that failed: the publish returned an error and the task is blocked. [ACORN-2](/ACORN/issues/ACORN-2) went through and is done.", "| Task | Outcome |\n|---|---|\n| [ACORN-1](/ACORN/issues/ACORN-1) | Blocked - publish error, nothing was published |\n| [ACORN-2](/ACORN/issues/ACORN-2) | Done, published once |", "ACORN-1 didn't succeed: the publish failed. ACORN-2 finished and its update was published."]) @@ -143,6 +145,7 @@ describe("per-task outcomes are graded against recorded status", () => { "The routine ran twice: ACORN-1 and ACORN-2. Both runs completed successfully.", "ACORN-1 failed. ACORN-2 failed too, although its run completed.", "ACORN-1 failed. The other run worked.", + RECORDED_2.replace("[ACORN-1](/ACORN/issues/ACORN-1) remains blocked", "[ACORN-1](/ACORN/issues/ACORN-1) is done"), "Runs this week: [ACORN-1](/ACORN/issues/ACORN-1), [ACORN-2](/ACORN/issues/ACORN-2)."]) expect(withReply(reply), reply).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts index 32e01df8e49..e1159fefeef 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -42,9 +42,14 @@ const FAILURE = new RegExp(`${negated}\\b(?:fail(?:ed|s|ure)?|unsuccessful|errou * completed invocation): the reply must speak about each task, give it its recorded outcome, and never give it the * other one. Saved status, not wording, is what makes an outcome right; the vocabulary only detects which one is claimed. */ export function preservesRecordedOutcomes(reply: string, failedTask: Entity, completedTask: Entity) { - const units = taskOutcomeUnits(reply, [failedTask, completedTask]); + // Quoted tool output (`success: false`, "Synthetic write failed") is data the reply cites, not its own claim. + const prose = reply.replace(/`([^`\n]*)`|"([^"\n]*)"|“([^”\n]*)”/g, (_quote, ...spans: Array) => { + const text = spans.slice(0, 3).find(span => span !== undefined) ?? ""; + return /^[A-Z][A-Z0-9]*-\d+$/.test(text.trim()) ? text : " "; + }); + const units = taskOutcomeUnits(prose, [failedTask, completedTask]); const failed = units.get(failedTask.id) ?? [], completed = units.get(completedTask.id) ?? []; - const blanket = reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence)); + const blanket = prose.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence)); return failed.some(unit => FAILURE.test(unit)) && completed.some(unit => SUCCESS.test(unit)) && !failed.some(unit => SUCCESS.test(unit)) && !completed.some(unit => FAILURE.test(unit)) && !blanket; } @@ -118,5 +123,5 @@ export const integratorRoutineHistoryCases = [integratorDefinition("routines", " const routine = state.routine as RoutineDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title); }, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/routine-history-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"], - oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v3", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000, + oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v4", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000, })];