test(evals): preserve routine invocation and native failure history

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { gradeIntegratorRoutineHistory } from "./integrator-routine-history.js";
+function valid() {
+  const first = { id: "failed-task", identifier: "ACORN-1", title: "Morning Launch summary", status: "in_progress" };
+  const second = { id: "completed-task", identifier: "ACORN-2", title: "Morning Launch summary", status: "done" };
+  const failure = { id: "failed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T12:00:00Z", errorCode: "provider_error", error: "Unavailable model", contextSnapshot: { issueId: first.id }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "unavailable-test" } } } };
+  return { scenarioId: "routines.read-history", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { worker: { id: "morgan" }, failedIssueId: first.id, completedIssueId: second.id,
+    failedInvocationId: "invocation-one", completedInvocationId: "invocation-two", failedRun: structuredClone(failure), failedModel: "unavailable-test", failedEvents: [{ eventType: "run.terminal" }], completedNativeRunId: "completed-native", weekStart: Date.parse("2026-10-26T00:00:00Z"), weekEnd: Date.parse("2026-11-02T00:00:00Z") },
+    tasks: [first, second], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: ["/ACORN/issues/ACORN-1", "/ACORN/issues/ACORN-2"],
+    runs: [failure, { id: "completed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: second.id } }],
+    routine: { id: "routine", title: "Morning Launch summary" }, executions: [{ id: "invocation-one", source: "manual", status: "issue_created", linkedIssueId: first.id, triggeredAt: "2026-10-26T12:00:00Z" }, { id: "invocation-two", source: "manual", status: "completed", linkedIssueId: second.id, triggeredAt: "2026-10-26T13:00:00Z" }],
+    reply: "ACORN-1: its native execution failed; the routine invocation is still issue_created.\nACORN-2: the execution completed successfully.\nBoth are manual runs this week; there is no confirmed scheduled morning execution." };
+}
+describe("routine invocation and native failure distinction", () => {
+  it("accepts distinct genuinely admitted failure and completed invocation, never absent evidence", () => {
+    expect(gradeIntegratorRoutineHistory(valid()).passed).toBe(true);
+    expect(gradeIntegratorRoutineHistory(undefined).passed).toBe(false);
+    const missing = valid(); missing.state.failedEvents = [];
+    expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false);
+  });
+  it("rejects preflight or cancellation substituted for an admitted native failure", () => {
+    const cancelled = valid(); cancelled.runs[0]!.status = "cancelled";
+    expect(gradeIntegratorRoutineHistory(cancelled).passed).toBe(false);
+    const wrongModel = valid(); wrongModel.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.model = "valid-model";
+    expect(gradeIntegratorRoutineHistory(wrongModel).passed).toBe(false);
+    const wrongTask = valid(); wrongTask.runs[1]!.contextSnapshot.issueId = "unrelated-task";
+    expect(gradeIntegratorRoutineHistory(wrongTask).passed).toBe(false);
+    const stale = valid(); stale.executions[0]!.triggeredAt = "2026-10-19T12:00:00Z";
+    expect(gradeIntegratorRoutineHistory(stale).passed).toBe(false);
+  });
+  it("rejects last-success-only, false scheduled success and missing failure links", () => {
+    const o = valid();
+    expect(gradeIntegratorRoutineHistory({ ...o, reply: "ACORN-2 completed successfully; all runs succeeded this week. The saved schedule is proof." }).passed).toBe(false);
+    expect(gradeIntegratorRoutineHistory({ ...o, reply: "ACORN-1 failed. ACORN-2 completed. Both scheduled morning summaries ran successfully." }).passed).toBe(false);
+    expect(gradeIntegratorRoutineHistory({ ...o, links: ["/ACORN/issues/ACORN-2"] }).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts

@@ -0,0 +1,70 @@
+import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { pollUntil } from "../api.js";
+import { checks, hasLink, observation, project, type Entity, type InternalObservation } from "./internal-support.js";
+import { fixtureWorker } from "./internal-definition.js";
+import { unavailableNativeModelFailure, admittedModelFailure } from "./runtime-native-failure.js";
+import type { RuntimeAgent } from "./runtime-support.js";
+import { groundedStatement, integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js";
+interface HistoryObservation extends InternalObservation { routine: RoutineDetail; executions: RoutineRunSummary[] }
+const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
+async function observeHistory(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<HistoryObservation> {
+  const base = await observeInternal(ctx, state); const routine = state.routine as RoutineDetail;
+  return { ...base, routine: await ctx.input.api.get<RoutineDetail>(`/api/routines/${routine.id}`), executions: await ctx.input.api.get<RoutineRunSummary[]>(`/api/routines/${routine.id}/runs`) };
+}
+function historyObservation(value: unknown): HistoryObservation | undefined {
+  const o = value as HistoryObservation;
+  return observation(value) && !!o.routine && Array.isArray(o.executions) ? o : undefined;
+}
+const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+export function gradeIntegratorRoutineHistory(value: unknown) {
+  const c = checks(); const o = historyObservation(value);
+  c.add("actual-routine-and-native-execution-history", !!o, "Requires real public invocation history plus distinct actual native failure and successful execution receipts.");
+  if (!o) return c.result();
+  const failedTask = o.tasks.find(task => task.id === o.state.failedIssueId); const completedTask = o.tasks.find(task => task.id === o.state.completedIssueId);
+  const failedInvocation = o.executions.find(run => run.id === o.state.failedInvocationId); const completedInvocation = o.executions.find(run => run.id === o.state.completedInvocationId);
+  const failed = o.runs.find(run => run.id === (o.state.failedRun as { id?: string } | undefined)?.id);
+  const failureEvidence = o.state.failedRun as Parameters<typeof admittedModelFailure>[0]["run"];
+  c.add("genuine-task-bound-native-failure", !!failedTask && failed?.status === "failed" && failed.runtimeMode === "native" && !!failedInvocation && failedInvocation.linkedIssueId === failedTask.id
+    && admittedModelFailure({ run: failureEvidence, events: o.state.failedEvents as Parameters<typeof admittedModelFailure>[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status.");
+  c.add("separate-real-successful-invocation", !!completedTask && completedTask.status === "done" && completedInvocation?.status === "completed" && completedInvocation.linkedIssueId === completedTask.id
+    && completedTask.id !== failedTask?.id && o.runs.some(run => run.id === o.state.completedNativeRunId && run.status === "succeeded" && run.runtimeMode === "native" && run.contextSnapshot?.issueId === completedTask.id), "A second independently invoked routine actually completed its normal task under a successful native run.");
+  const start = Number(o.state.weekStart), end = Number(o.state.weekEnd);
+  c.add("requested-week-and-exact-history-retained", [failedInvocation, completedInvocation].every(run => !!run && Date.parse(String(run.triggeredAt)) >= start && Date.parse(String(run.triggeredAt)) < end)
+    && o.executions.length === 2 && failedInvocation?.source === "manual" && completedInvocation?.source === "manual", "Both actual manual invocations fall inside the measured current week, with no hidden or duplicate history rows.");
+  c.add("actual-failure-and-completion-linked", !!failedTask && !!completedTask && hasLink(o, "issues", failedTask) && hasLink(o, "issues", completedTask), "The answer links both real execution tasks.");
+  c.add("failed-native-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier
+    && groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i"))
+    && groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i)
+    && /\bmanual\b/i.test(o.reply) && /\b(?:scheduled|schedule)\b/i.test(o.reply) && /\b(?:no (?:recorded |confirmed )?scheduled|not (?:proof|evidence)|cannot confirm|can't confirm|missing)\b/i.test(o.reply), "The response preserves which actual native execution failed and which invocation completed, identifies the manual sources, and does not infer scheduled morning success from a saved schedule.");
+  return c.result();
+}
+export const integratorRoutineHistoryCases = [integratorDefinition("routines", "routines.read-history", async ctx => {
+  const p = await project(ctx, "Launch", { description: "Synthetic Launch project: payment approval remains pending; no other active work." }); const worker = await fixtureWorker(ctx);
+  for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) await ctx.input.api.post(`${root(ctx)}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
+  const routine = await ctx.input.api.post<RoutineDetail>(`${root(ctx)}/routines`, { title: "Morning Launch summary", projectId: p.id, assigneeAgentId: worker.id, concurrencyPolicy: "always_enqueue", description: "Summarize the actual Launch project in a short saved task comment, link the project, and mark this summary task done. Do not edit project work or create follow-up tasks." });
+  let failedInvocation: RoutineRunSummary | undefined;
+  const failure = await unavailableNativeModelFailure(ctx, worker as RuntimeAgent, async () => {
+    failedInvocation = await ctx.input.api.post<RoutineRunSummary>(`/api/routines/${routine.id}/run`, { source: "manual", idempotencyKey: `failed-history-${ctx.input.nonce}` });
+    if (!failedInvocation.linkedIssueId) throw new Error("Routine failure prerequisite did not admit an actual execution task");
+    return { issueId: failedInvocation.linkedIssueId };
+  });
+  const completedInvocation = await ctx.input.api.post<RoutineRunSummary>(`/api/routines/${routine.id}/run`, { source: "manual", idempotencyKey: `success-history-${ctx.input.nonce}` });
+  if (!completedInvocation.linkedIssueId) throw new Error("Successful routine prerequisite did not create a real execution task");
+  const success = await pollUntil({ label: "actual successful routine history", deadlineAt: Date.now() + 180_000, intervalMs: 1000,
+    load: async () => ({ runs: await ctx.allRuns(), history: await ctx.input.api.get<RoutineRunSummary[]>(`/api/routines/${routine.id}/runs`) }),
+    accept: value => value.runs.length === 2 && value.runs.some(run => run.status === "succeeded" && run.contextSnapshot?.issueId === completedInvocation.linkedIssueId)
+      && value.history.some(run => run.id === completedInvocation.id && run.status === "completed"),
+    reject: value => value.runs.length > 2 ? "Routine history fixture exceeded its two declared real native invocations" : undefined });
+  const weekStart = new Date(); weekStart.setUTCHours(0, 0, 0, 0); weekStart.setUTCDate(weekStart.getUTCDate() - (weekStart.getUTCDay() + 6) % 7);
+  return { project: p, worker: { id: worker.id }, routine, failedInvocationId: failedInvocation!.id, completedInvocationId: completedInvocation.id, failedIssueId: failure.failedIssueId, completedIssueId: completedInvocation.linkedIssueId,
+    failedModel: failure.failedModel, failedRun: { ...failure.failedRun, runnerProfileJson: { nativeExecutionInput: { provider: failure.failedRun.runnerProfileJson?.nativeExecutionInput?.provider } } },
+    failedEvents: failure.failedRunEvents.map(event => ({ seq: event.seq, eventType: event.eventType, payload: { prpEvent: { eventType: event.payload?.prpEvent?.eventType } } })),
+    completedNativeRunId: success.runs.find(run => run.contextSnapshot?.issueId === completedInvocation.linkedIssueId)?.id,
+    weekStart: weekStart.getTime(), weekEnd: weekStart.getTime() + 7 * 86400_000 };
+}, observeHistory, gradeIntegratorRoutineHistory, async (ctx, state) => {
+  const routine = state.routine as RoutineDetail;
+  await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title);
+}, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"],
+  expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } }, attemptTimeoutMs: 720_000,
+})];