diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts new file mode 100644 index 00000000000..67fcd415281 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { gradeIntegratorRoutineHistory } from "./integrator-routine-history.js"; +function valid() { + const first = { id: "failed-task", identifier: "ACORN-1", title: "Morning Launch summary", status: "in_progress" }; + const second = { id: "completed-task", identifier: "ACORN-2", title: "Morning Launch summary", status: "done" }; + const failure = { id: "failed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T12:00:00Z", errorCode: "provider_error", error: "Unavailable model", contextSnapshot: { issueId: first.id }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "unavailable-test" } } } }; + return { scenarioId: "routines.read-history", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { worker: { id: "morgan" }, failedIssueId: first.id, completedIssueId: second.id, + failedInvocationId: "invocation-one", completedInvocationId: "invocation-two", failedRun: structuredClone(failure), failedModel: "unavailable-test", failedEvents: [{ eventType: "run.terminal" }], completedNativeRunId: "completed-native", weekStart: Date.parse("2026-10-26T00:00:00Z"), weekEnd: Date.parse("2026-11-02T00:00:00Z") }, + tasks: [first, second], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: ["/ACORN/issues/ACORN-1", "/ACORN/issues/ACORN-2"], + runs: [failure, { id: "completed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: second.id } }], + routine: { id: "routine", title: "Morning Launch summary" }, executions: [{ id: "invocation-one", source: "manual", status: "issue_created", linkedIssueId: first.id, triggeredAt: "2026-10-26T12:00:00Z" }, { id: "invocation-two", source: "manual", status: "completed", linkedIssueId: second.id, triggeredAt: "2026-10-26T13:00:00Z" }], + reply: "ACORN-1: its native execution failed; the routine invocation is still issue_created.\nACORN-2: the execution completed successfully.\nBoth are manual runs this week; there is no confirmed scheduled morning execution." }; +} +describe("routine invocation and native failure distinction", () => { + it("accepts distinct genuinely admitted failure and completed invocation, never absent evidence", () => { + expect(gradeIntegratorRoutineHistory(valid()).passed).toBe(true); + expect(gradeIntegratorRoutineHistory(undefined).passed).toBe(false); + const missing = valid(); missing.state.failedEvents = []; + expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false); + }); + it("rejects preflight or cancellation substituted for an admitted native failure", () => { + const cancelled = valid(); cancelled.runs[0]!.status = "cancelled"; + expect(gradeIntegratorRoutineHistory(cancelled).passed).toBe(false); + const wrongModel = valid(); wrongModel.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.model = "valid-model"; + expect(gradeIntegratorRoutineHistory(wrongModel).passed).toBe(false); + const wrongTask = valid(); wrongTask.runs[1]!.contextSnapshot.issueId = "unrelated-task"; + expect(gradeIntegratorRoutineHistory(wrongTask).passed).toBe(false); + const stale = valid(); stale.executions[0]!.triggeredAt = "2026-10-19T12:00:00Z"; + expect(gradeIntegratorRoutineHistory(stale).passed).toBe(false); + }); + it("rejects last-success-only, false scheduled success and missing failure links", () => { + const o = valid(); + expect(gradeIntegratorRoutineHistory({ ...o, reply: "ACORN-2 completed successfully; all runs succeeded this week. The saved schedule is proof." }).passed).toBe(false); + expect(gradeIntegratorRoutineHistory({ ...o, reply: "ACORN-1 failed. ACORN-2 completed. Both scheduled morning summaries ran successfully." }).passed).toBe(false); + expect(gradeIntegratorRoutineHistory({ ...o, links: ["/ACORN/issues/ACORN-2"] }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts new file mode 100644 index 00000000000..bb539da2cab --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -0,0 +1,70 @@ +import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, hasLink, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { unavailableNativeModelFailure, admittedModelFailure } from "./runtime-native-failure.js"; +import type { RuntimeAgent } from "./runtime-support.js"; +import { groundedStatement, integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +interface HistoryObservation extends InternalObservation { routine: RoutineDetail; executions: RoutineRunSummary[] } +const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; +async function observeHistory(ctx: ChatCatalogContext, state: Record): Promise { + const base = await observeInternal(ctx, state); const routine = state.routine as RoutineDetail; + return { ...base, routine: await ctx.input.api.get(`/api/routines/${routine.id}`), executions: await ctx.input.api.get(`/api/routines/${routine.id}/runs`) }; +} +function historyObservation(value: unknown): HistoryObservation | undefined { + const o = value as HistoryObservation; + return observation(value) && !!o.routine && Array.isArray(o.executions) ? o : undefined; +} +const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +export function gradeIntegratorRoutineHistory(value: unknown) { + const c = checks(); const o = historyObservation(value); + c.add("actual-routine-and-native-execution-history", !!o, "Requires real public invocation history plus distinct actual native failure and successful execution receipts."); + if (!o) return c.result(); + const failedTask = o.tasks.find(task => task.id === o.state.failedIssueId); const completedTask = o.tasks.find(task => task.id === o.state.completedIssueId); + const failedInvocation = o.executions.find(run => run.id === o.state.failedInvocationId); const completedInvocation = o.executions.find(run => run.id === o.state.completedInvocationId); + const failed = o.runs.find(run => run.id === (o.state.failedRun as { id?: string } | undefined)?.id); + const failureEvidence = o.state.failedRun as Parameters[0]["run"]; + c.add("genuine-task-bound-native-failure", !!failedTask && failed?.status === "failed" && failed.runtimeMode === "native" && !!failedInvocation && failedInvocation.linkedIssueId === failedTask.id + && admittedModelFailure({ run: failureEvidence, events: o.state.failedEvents as Parameters[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status."); + c.add("separate-real-successful-invocation", !!completedTask && completedTask.status === "done" && completedInvocation?.status === "completed" && completedInvocation.linkedIssueId === completedTask.id + && completedTask.id !== failedTask?.id && o.runs.some(run => run.id === o.state.completedNativeRunId && run.status === "succeeded" && run.runtimeMode === "native" && run.contextSnapshot?.issueId === completedTask.id), "A second independently invoked routine actually completed its normal task under a successful native run."); + const start = Number(o.state.weekStart), end = Number(o.state.weekEnd); + c.add("requested-week-and-exact-history-retained", [failedInvocation, completedInvocation].every(run => !!run && Date.parse(String(run.triggeredAt)) >= start && Date.parse(String(run.triggeredAt)) < end) + && o.executions.length === 2 && failedInvocation?.source === "manual" && completedInvocation?.source === "manual", "Both actual manual invocations fall inside the measured current week, with no hidden or duplicate history rows."); + c.add("actual-failure-and-completion-linked", !!failedTask && !!completedTask && hasLink(o, "issues", failedTask) && hasLink(o, "issues", completedTask), "The answer links both real execution tasks."); + c.add("failed-native-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier + && groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i")) + && groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i) + && /\bmanual\b/i.test(o.reply) && /\b(?:scheduled|schedule)\b/i.test(o.reply) && /\b(?:no (?:recorded |confirmed )?scheduled|not (?:proof|evidence)|cannot confirm|can't confirm|missing)\b/i.test(o.reply), "The response preserves which actual native execution failed and which invocation completed, identifies the manual sources, and does not infer scheduled morning success from a saved schedule."); + return c.result(); +} +export const integratorRoutineHistoryCases = [integratorDefinition("routines", "routines.read-history", async ctx => { + const p = await project(ctx, "Launch", { description: "Synthetic Launch project: payment approval remains pending; no other active work." }); const worker = await fixtureWorker(ctx); + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) await ctx.input.api.post(`${root(ctx)}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const routine = await ctx.input.api.post(`${root(ctx)}/routines`, { title: "Morning Launch summary", projectId: p.id, assigneeAgentId: worker.id, concurrencyPolicy: "always_enqueue", description: "Summarize the actual Launch project in a short saved task comment, link the project, and mark this summary task done. Do not edit project work or create follow-up tasks." }); + let failedInvocation: RoutineRunSummary | undefined; + const failure = await unavailableNativeModelFailure(ctx, worker as RuntimeAgent, async () => { + failedInvocation = await ctx.input.api.post(`/api/routines/${routine.id}/run`, { source: "manual", idempotencyKey: `failed-history-${ctx.input.nonce}` }); + if (!failedInvocation.linkedIssueId) throw new Error("Routine failure prerequisite did not admit an actual execution task"); + return { issueId: failedInvocation.linkedIssueId }; + }); + const completedInvocation = await ctx.input.api.post(`/api/routines/${routine.id}/run`, { source: "manual", idempotencyKey: `success-history-${ctx.input.nonce}` }); + if (!completedInvocation.linkedIssueId) throw new Error("Successful routine prerequisite did not create a real execution task"); + const success = await pollUntil({ label: "actual successful routine history", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: async () => ({ runs: await ctx.allRuns(), history: await ctx.input.api.get(`/api/routines/${routine.id}/runs`) }), + accept: value => value.runs.length === 2 && value.runs.some(run => run.status === "succeeded" && run.contextSnapshot?.issueId === completedInvocation.linkedIssueId) + && value.history.some(run => run.id === completedInvocation.id && run.status === "completed"), + reject: value => value.runs.length > 2 ? "Routine history fixture exceeded its two declared real native invocations" : undefined }); + const weekStart = new Date(); weekStart.setUTCHours(0, 0, 0, 0); weekStart.setUTCDate(weekStart.getUTCDate() - (weekStart.getUTCDay() + 6) % 7); + return { project: p, worker: { id: worker.id }, routine, failedInvocationId: failedInvocation!.id, completedInvocationId: completedInvocation.id, failedIssueId: failure.failedIssueId, completedIssueId: completedInvocation.linkedIssueId, + failedModel: failure.failedModel, failedRun: { ...failure.failedRun, runnerProfileJson: { nativeExecutionInput: { provider: failure.failedRun.runnerProfileJson?.nativeExecutionInput?.provider } } }, + failedEvents: failure.failedRunEvents.map(event => ({ seq: event.seq, eventType: event.eventType, payload: { prpEvent: { eventType: event.payload?.prpEvent?.eventType } } })), + completedNativeRunId: success.runs.find(run => run.contextSnapshot?.issueId === completedInvocation.linkedIssueId)?.id, + weekStart: weekStart.getTime(), weekEnd: weekStart.getTime() + 7 * 86400_000 }; +}, observeHistory, gradeIntegratorRoutineHistory, async (ctx, state) => { + const routine = state.routine as RoutineDetail; + await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title); +}, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"], + expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 2 } }, attemptTimeoutMs: 720_000, +})];