From 031aab40410a3e162b9ca7c72712a8b39f51b8a4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 21:27:18 -0600 Subject: [PATCH] test(evals): fail retire-agent setup when the historical record starts work The baseline and second runs both had a third Morgan run, queued by creating the done historical task. Assert in setup that Morgan's only run is the active assignment, so this shows up as a setup defect, not a candidate one. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 3 ++- tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index be0ff82a0f8..a1a5b56b3f3 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -790,7 +790,8 @@ "invariantCheckIds": [ "lead-ready", "real-active-assignment", - "truthful-historical-work" + "truthful-historical-work", + "historical-record-starts-no-work" ] }, "runPolicy": { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts index 3a6fefb3e08..b75e23e435e 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts @@ -47,7 +47,7 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", sourceFiles: ["chat-catalog-domains/runtime-retirement.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], expectedRunCount: 2, attemptTimeoutMs: 600_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } }, - setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-active-assignment", "truthful-historical-work"] }, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-active-assignment", "truthful-historical-work", "historical-record-starts-no-work"] }, async verifySetup(ctx, state) { const c = checks(); const worker = state.worker as RuntimeAgent; const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); @@ -61,6 +61,9 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", c.add("truthful-historical-work", historical.status === "done" && historical.assigneeAgentId === worker.id && historical.companyId === ctx.input.fixtures.company.id && !historical.createdByAgentId && !historical.createdByRunId, "The independently read prior completed record has genuine human provenance, without forged model history."); + const workerRuns = (await ctx.allRuns()).filter(row => row.agentId === worker.id); + c.add("historical-record-starts-no-work", workerRuns.length === 1 && workerRuns[0]?.id === state.workerRunId, + "Recording finished work starts no run: Morgan's only run is the active assignment."); return c.result(); }, async act(ctx, state) {