diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts index e6633d10d53..89e8a9056bd 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -4,7 +4,7 @@ function valid() { const first = { id: "failed-task", identifier: "ACORN-1", title: "Morning Launch summary", status: "in_progress" }; const second = { id: "completed-task", identifier: "ACORN-2", title: "Morning Launch summary", status: "done" }; const failure = { id: "failed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T12:00:00Z", errorCode: "provider_error", error: "Unavailable model", contextSnapshot: { issueId: first.id }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "unavailable-test" } } } }; - return { scenarioId: "routines.read-history", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { worker: { id: "morgan" }, failedIssueId: first.id, completedIssueId: second.id, + return { expectedProvider: "codex", scenarioId: "routines.read-history", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { worker: { id: "morgan" }, failedIssueId: first.id, completedIssueId: second.id, failedInvocationId: "invocation-one", completedInvocationId: "invocation-two", failedRun: structuredClone(failure), failedModel: "unavailable-test", failedEvents: [{ eventType: "run.terminal" }], completedNativeRunId: "completed-native", weekStart: Date.parse("2026-10-26T00:00:00Z"), weekEnd: Date.parse("2026-11-02T00:00:00Z") }, tasks: [first, second], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: ["/ACORN/issues/ACORN-1", "/ACORN/issues/ACORN-2"], runs: [failure, { id: "completed-native", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: second.id } }], @@ -20,10 +20,16 @@ describe("routine invocation and native failure distinction", () => { expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false); }); it("accepts actual OpenCode failure history instead of defaulting its provider to Codex", () => { - const o = valid(); + const o = valid(); o.expectedProvider = "opencode"; o.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = "opencode"; o.runs[0] = structuredClone(o.state.failedRun); expect(gradeIntegratorRoutineHistory(o).passed).toBe(true); + const wrongBoth = structuredClone(o); + wrongBoth.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = "codex"; + wrongBoth.runs[0] = structuredClone(wrongBoth.state.failedRun); + expect(gradeIntegratorRoutineHistory(wrongBoth).passed).toBe(false); + const missingExpected = structuredClone(o); missingExpected.expectedProvider = ""; + expect(gradeIntegratorRoutineHistory(missingExpected).passed).toBe(false); const wrongProvider = structuredClone(o); wrongProvider.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = "codex"; expect(gradeIntegratorRoutineHistory(wrongProvider).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts index 491def144f2..259947425c3 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -8,11 +8,11 @@ import { fixtureWorker } from "./internal-definition.js"; import { unavailableNativeModelFailure, admittedModelFailure } from "./runtime-native-failure.js"; import type { RuntimeAgent } from "./runtime-support.js"; import { groundedStatement, integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; -interface HistoryObservation extends InternalObservation { routine: RoutineDetail; executions: RoutineRunSummary[] } +interface HistoryObservation extends InternalObservation { expectedProvider: string; routine: RoutineDetail; executions: RoutineRunSummary[] } const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; async function observeHistory(ctx: ChatCatalogContext, state: Record): Promise { const base = await observeInternal(ctx, state); const routine = state.routine as RoutineDetail; - return { ...base, routine: await ctx.input.api.get(`/api/routines/${routine.id}`), executions: await ctx.input.api.get(`/api/routines/${routine.id}/runs`) }; + return { ...base, expectedProvider: ctx.input.execution.profile.provider, routine: await ctx.input.api.get(`/api/routines/${routine.id}`), executions: await ctx.input.api.get(`/api/routines/${routine.id}/runs`) }; } function historyObservation(value: unknown): HistoryObservation | undefined { const o = value as HistoryObservation; @@ -27,9 +27,10 @@ export function gradeIntegratorRoutineHistory(value: unknown) { const failedInvocation = o.executions.find(run => run.id === o.state.failedInvocationId); const completedInvocation = o.executions.find(run => run.id === o.state.completedInvocationId); const failed = o.runs.find(run => run.id === (o.state.failedRun as { id?: string } | undefined)?.id); const failureEvidence = o.state.failedRun as Parameters[0]["run"]; - const provider = (failed as typeof failureEvidence)?.runnerProfileJson?.nativeExecutionInput?.provider?.kind; + const provider = o.expectedProvider; + const observedProvider = (failed as typeof failureEvidence)?.runnerProfileJson?.nativeExecutionInput?.provider?.kind; c.add("genuine-task-bound-native-failure", !!failedTask && failed?.status === "failed" && failed.runtimeMode === "native" && !!failedInvocation && failedInvocation.linkedIssueId === failedTask.id - && (provider === "codex" || provider === "opencode") && admittedModelFailure({ provider, run: failureEvidence, events: o.state.failedEvents as Parameters[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status."); + && (provider === "codex" || provider === "opencode") && observedProvider === provider && admittedModelFailure({ provider, run: failureEvidence, events: o.state.failedEvents as Parameters[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status."); c.add("separate-real-successful-invocation", !!completedTask && completedTask.status === "done" && completedInvocation?.status === "completed" && completedInvocation.linkedIssueId === completedTask.id && completedTask.id !== failedTask?.id && o.runs.some(run => run.id === o.state.completedNativeRunId && run.status === "succeeded" && run.runtimeMode === "native" && run.contextSnapshot?.issueId === completedTask.id), "A second independently invoked routine actually completed its normal task under a successful native run."); const start = Number(o.state.weekStart), end = Number(o.state.weekEnd);