From 78783dda578ab8a04e337eba2c3a97f7a3204768 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 00:47:54 -0500 Subject: [PATCH] fix(evals): bind routine history grading to observed provider --- .../integrator-routine-history.test.ts | 17 +++++++++++++++++ .../integrator-routine-history.ts | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts index 8d587aa63db..e6633d10d53 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts @@ -19,6 +19,23 @@ describe("routine invocation and native failure distinction", () => { const missing = valid(); missing.state.failedEvents = []; expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false); }); + it("accepts actual OpenCode failure history instead of defaulting its provider to Codex", () => { + const o = valid(); + o.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = "opencode"; + o.runs[0] = structuredClone(o.state.failedRun); + expect(gradeIntegratorRoutineHistory(o).passed).toBe(true); + const wrongProvider = structuredClone(o); + wrongProvider.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = "codex"; + expect(gradeIntegratorRoutineHistory(wrongProvider).passed).toBe(false); + const missingProvider = structuredClone(o); + missingProvider.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.kind = ""; + expect(gradeIntegratorRoutineHistory(missingProvider).passed).toBe(false); + const wrongModel = structuredClone(o); + wrongModel.state.failedRun.runnerProfileJson.nativeExecutionInput.provider.model = "supported-model"; + expect(gradeIntegratorRoutineHistory(wrongModel).passed).toBe(false); + const noTerminal = structuredClone(o); noTerminal.state.failedEvents = []; + expect(gradeIntegratorRoutineHistory(noTerminal).passed).toBe(false); + }); it("rejects preflight or cancellation substituted for an admitted native failure", () => { const cancelled = valid(); cancelled.runs[0]!.status = "cancelled"; expect(gradeIntegratorRoutineHistory(cancelled).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts index 87b8a53bd9d..491def144f2 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts @@ -27,8 +27,9 @@ export function gradeIntegratorRoutineHistory(value: unknown) { const failedInvocation = o.executions.find(run => run.id === o.state.failedInvocationId); const completedInvocation = o.executions.find(run => run.id === o.state.completedInvocationId); const failed = o.runs.find(run => run.id === (o.state.failedRun as { id?: string } | undefined)?.id); const failureEvidence = o.state.failedRun as Parameters[0]["run"]; + const provider = (failed as typeof failureEvidence)?.runnerProfileJson?.nativeExecutionInput?.provider?.kind; c.add("genuine-task-bound-native-failure", !!failedTask && failed?.status === "failed" && failed.runtimeMode === "native" && !!failedInvocation && failedInvocation.linkedIssueId === failedTask.id - && admittedModelFailure({ run: failureEvidence, events: o.state.failedEvents as Parameters[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status."); + && (provider === "codex" || provider === "opencode") && admittedModelFailure({ provider, run: failureEvidence, events: o.state.failedEvents as Parameters[0]["events"] ?? [], companyId: o.companyId, agentId: (o.state.worker as Entity | undefined)?.id ?? "", issueId: failedTask.id, model: String(o.state.failedModel) }), "The failure is a genuinely admitted task-bound unavailable-model native execution, never a rejected HTTP request or a patched/cancelled status."); c.add("separate-real-successful-invocation", !!completedTask && completedTask.status === "done" && completedInvocation?.status === "completed" && completedInvocation.linkedIssueId === completedTask.id && completedTask.id !== failedTask?.id && o.runs.some(run => run.id === o.state.completedNativeRunId && run.status === "succeeded" && run.runtimeMode === "native" && run.contextSnapshot?.issueId === completedTask.id), "A second independently invoked routine actually completed its normal task under a successful native run."); const start = Number(o.state.weekStart), end = Number(o.state.weekEnd);