test(evals): use production chat timeout within finite cell budgets

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -3070,7 +3070,14 @@ requests the existing owned supervisor's shutdown before further API observation
 Multiple probe receipts fail health. Setup-probe usage/cost is unavailable, never
 zero. This diagnostic cannot qualify a catalog scenario or rewrite prior results.
 
-### Model-spend readiness and operating reconciliation
+The manual `chat-features` suite uses the production default of no native active-turn
+cap (`timeoutSec: 0`). Zero describes only that per-turn cap; it does not mean zero
+cost or an unlimited cell. Existing finite scenario, Playwright, launcher watchdog,
+public cancellation and owned-process cleanup bounds remain in force. Setup evidence
+records the effective saved policy separately from authored legacy turn declarations.
+Earlier results with the 120-second cap remain unchanged historical measurements.
+
+# Model-spend readiness and operating reconciliation
 
 A reviewed operating allowance can opt into `accountingReadiness: "model_requests"`.
 This policy requires at least one model request and complete final model-cost

tests/runner-e2e/catalog.ts

@@ -1,3 +1,4 @@
+import { CHAT_NATIVE_TIMEOUT_POLICY } from "./chat-native-timeout-policy.js";
 import { CHAT_SETUP_HEALTH_SUITE, chatSetupHealthTask, chatSetupHealthDigest } from "./chat-setup-health.js";
 import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js";
 import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";
@@ -1626,7 +1627,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     groups: ["chat", "native"],
     profiles: [chatFeatureExecutionProfile],
     environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length,
-    definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only",
+    definitionMetadata: { version: 3, nativeTimeoutPolicy: CHAT_NATIVE_TIMEOUT_POLICY, authoredTurnTimeouts: "legacy-declarations-retained-separately-not-effective-native-caps", definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only",
       instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS,
       unpricedUsagePolicy: "allow-through-public-budget-policy", taskRunLimits: "per-case-declared; agent daily admission ceiling 3",
       maximumAttemptsPerCell: 1, setupProbeUsage: "unavailable-outside-heartbeat-accounting",

tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts

@@ -1,3 +1,4 @@
+import { CHAT_NATIVE_TIMEOUT_POLICY } from "../chat-native-timeout-policy.js";
 import { expect } from "@playwright/test";
 import { createHash } from "node:crypto";
 import { collectRunEvents } from "../run-observations.js";
@@ -11,8 +12,8 @@ interface PreparedAgent extends Entity { lifecycleState?: string; adapterConfig?
 export function gradeProjectDefaultsSetup(value: { lead: PreparedAgent; future: PreparedAgent; held: PreparedAgent; task: Entity & { executionRunId?: string }; runs: Entity[]; activeRunId: string; gateCalls: { authorized?: boolean }[] }) {
   const c = checks();
   const ready = (agent: PreparedAgent) => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status ?? "");
-  c.add("lead-ready", ready(value.lead) && value.lead.adapterConfig?.timeoutSec === 120, "The canonical lead is independently ready with its original 120-second native timeout.");
-  c.add("future-worker-ready", ready(value.future) && value.future.adapterConfig?.timeoutSec === 120, "The future-context worker is independently ready with its original 120-second native timeout.");
+  c.add("lead-ready", ready(value.lead) && value.lead.adapterConfig?.timeoutSec === CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, "The canonical lead is independently ready under the production-default native timeout policy and unchanged finite scenario budget.");
+  c.add("future-worker-ready", ready(value.future) && value.future.adapterConfig?.timeoutSec === CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, "The future-context worker is independently ready under the production-default native timeout policy and unchanged finite scenario budget.");
   c.add("existing-work-actively-owned", value.held.lifecycleState === "ready" && value.held.status === "running" && value.held.adapterConfig?.timeoutSec === 420
     && value.task.assigneeAgentId === value.held.id && value.task.executionRunId === value.activeRunId && value.runs.some(run => run.id === value.activeRunId && run.agentId === value.held.id && run.status === "running")
     && value.gateCalls.some(call => call.authorized), "Only the deliberately waiting prerequisite has the declared 420-second timeout, spanning the subsequent bounded 150+180-second stages with exact live ownership and authorized gate proof.");

tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts

@@ -6,9 +6,11 @@ import { gradeWorkTimeline } from "./internal-work-timeline.js";
 import { gradeArchivePartial } from "./internal-archive-partial.js";
 function base(scenarioId: string) { return { scenarioId, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: {} as Record<string, unknown>, tasks: [] as any[], projects: [] as any[], goals: [] as any[], comments: [] as any[], activity: [] as any[], documents: [], revisions: [], links: [] as string[], reply: "", humanQueries: [], runs: [] as any[] }; }
 it("preserves the exact active prerequisite while independently proving ready canonical and future agents", () => {
-  const ready = { id: "ceo", lifecycleState: "ready", status: "idle", adapterConfig: { timeoutSec: 120 } };
+  const ready = { id: "ceo", lifecycleState: "ready", status: "idle", adapterConfig: { timeoutSec: 0 } };
   const prepared = { lead: ready, future: { ...ready, id: "riley" }, held: { ...ready, id: "morgan", status: "running", adapterConfig: { timeoutSec: 420 } }, task: { id: "task", assigneeAgentId: "morgan", executionRunId: "held" }, runs: [{ id: "held", agentId: "morgan", status: "running" }], activeRunId: "held", gateCalls: [{ authorized: true }] };
   expect(gradeProjectDefaultsSetup(prepared).passed).toBe(true);
+  expect(gradeProjectDefaultsSetup({ ...prepared, lead: { ...prepared.lead, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false);
+  expect(gradeProjectDefaultsSetup({ ...prepared, future: { ...prepared.future, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false);
   expect(gradeProjectDefaultsSetup({ ...prepared, held: { ...prepared.held, status: "idle" } }).passed).toBe(false);
   expect(gradeProjectDefaultsSetup({ ...prepared, future: { ...prepared.future, status: "error" } }).passed).toBe(false);
   expect(gradeProjectDefaultsSetup({ ...prepared, held: { ...prepared.held, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false);

tests/runner-e2e/chat-catalog-flow.ts

@@ -1,3 +1,4 @@
+import { chatNativeTimeoutEvidence } from "./chat-native-timeout-policy.js";
 import { completeNativeSlackFileConditional, dispatchNativeSlackIngress, prepareNativeSlackOtherThread, slackOtherThreadBoundaryMatches, slackNativeRunCheck } from "./chat-slack-ingress.js";
 import { runScopedCompanionSurface } from "./chat-companion-surface.js";
 import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js";
@@ -103,9 +104,15 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
     await input.evidence("chat-catalog-funding.json", { scopes, mode: implementation.setupPolicy?.funding ?? "standard",
       policies: funding.policies.filter(policy => scopes.some(scope => scope.scopeType === policy.scopeType && scope.scopeId === policy.scopeId)), setupProbeUsage: "unknown; production hello probes are outside heartbeat accounting" });
     if (slack && implementation.slackIngress?.setupThread === "same-agent-distractor") await prepareNativeSlackOtherThread(context,implementation,state,deadlineAt);
-    await input.evidence("chat-catalog-setup.json", { scenario, state, inventoryDigest: chatCatalogInventoryDigest,
+    const savedLead = await input.api.get<{ adapterConfig?: { timeoutSec?: unknown } }>(`/api/agents/${input.fixtures.agent.id}`);
+    const nativeTimeoutPolicy = chatNativeTimeoutEvidence({ selectedTurnTimeoutMs: input.execution.task.turnTimeoutMs,
+      authoredLegacyTurnTimeoutMs: implementation.turnTimeoutMs, attemptTimeoutMs: implementation.attemptTimeoutMs,
+      observationBudgetMs: implementation.attemptTimeoutMs - 30_000,
+      savedAgentTimeoutSec: savedLead.adapterConfig?.timeoutSec });
+    await input.evidence("chat-catalog-setup.json", { nativeTimeoutPolicy, scenario, state, inventoryDigest: chatCatalogInventoryDigest,
       oracleVersion: implementation.oracleVersion, expectedRunCount: implementation.expectedRunCount,
-      minimumRunCount: implementation.minimumRunCount, turnTimeoutMs: implementation.turnTimeoutMs,
+      minimumRunCount: implementation.minimumRunCount, effectiveNativeTurnTimeoutMs: input.execution.task.turnTimeoutMs,
+      authoredLegacyTurnTimeoutMs: implementation.turnTimeoutMs,
       attemptTimeoutMs: implementation.attemptTimeoutMs });
     if (implementation.verifySetup) {
       const setupGrade = validateChatCatalogGrade(await implementation.verifySetup(context, state));

tests/runner-e2e/chat-feature-cases.ts

@@ -1,3 +1,4 @@
+import { CHAT_NATIVE_TIMEOUT_POLICY } from "./chat-native-timeout-policy.js";
 import { createHash } from "node:crypto";
 import { readFileSync, readdirSync } from "node:fs";
 import { parseDocument } from "yaml";
@@ -61,7 +62,7 @@ export function chatFeatureProfile(profile: RunnerProfileFixture): RunnerProfile
     return { ...payload, name: "CEO", title: "CEO", role: "ceo",
       capabilities: "Coordinates company work and delegates tasks to the team.",
       budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS,
-      adapterConfig: { ...adapterConfig, timeoutSec: 120, ...(profile.provider === "codex" ? { modelReasoningEffort: "low" } : {}) },
+      adapterConfig: { ...adapterConfig, timeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, ...(profile.provider === "codex" ? { modelReasoningEffort: "low" } : {}) },
       runtimeConfig: { ...(payload.runtimeConfig as Record<string, unknown>),
         heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: CHAT_FEATURE_BUDGET_CENTS } },
     };
@@ -71,7 +72,7 @@ export const chatFeatureTasks: readonly RunnerTaskFixture[] = CHAT_FEATURE_CASES
   id, label: id === "create-backlog" ? "Save a task for Morgan without starting it" : "Delegate an announcement and return its saved result",
   groups: ["chat"], flow: "agent_chat", workMode: "standard",
   expectedRunCount: id === "create-backlog" ? 1 : 3,
-  automaticRetryPolicy: "single_attempt", turnTimeoutMs: 120_000,
+  automaticRetryPolicy: "single_attempt", turnTimeoutMs: CHAT_NATIVE_TIMEOUT_POLICY.effectiveNativeTurnTimeoutMs,
   attemptTimeoutMs: { local: 420_000, daytona: 420_000 },
   expectedTerminalState: { issue: "in_review", run: "succeeded" },
   buildTitle: nonce => `Chat feature ${id} ${nonce}`,
@@ -88,7 +89,7 @@ export const chatCatalogTasks: readonly RunnerTaskFixture[] = chatCatalogCases.m
     requiredCredentials: implementation.requiredCredentials,
     id: scenario.id, label: scenario.id, groups: ["chat"], flow: "agent_chat", workMode: "standard",
     expectedRunCount: implementation.expectedRunCount, minimumExpectedRunCount: implementation.minimumRunCount,
-    automaticRetryPolicy: "single_attempt", turnTimeoutMs: implementation.turnTimeoutMs,
+    automaticRetryPolicy: "single_attempt", turnTimeoutMs: CHAT_NATIVE_TIMEOUT_POLICY.effectiveNativeTurnTimeoutMs,
     attemptTimeoutMs: { local: implementation.attemptTimeoutMs, daytona: implementation.attemptTimeoutMs },
     expectedTerminalState: { issue: "in_review", run: "succeeded" },
     buildTitle: nonce => `Chat catalog ${scenario.id} ${nonce}`,

tests/runner-e2e/chat-feature-flow.ts

@@ -1,3 +1,4 @@
+import { chatNativeTimeoutEvidence } from "./chat-native-timeout-policy.js";
 import { readChatReadinessReceipts } from "./chat-readiness-observer.js";
 import { legacyChatFeatureFingerprint } from "./chat-catalog-fingerprint.js";
 import { runChatCatalogFlow } from "./chat-catalog-flow.js";
@@ -108,7 +109,11 @@ export async function runChatFeatureFlow(context: {
       .map(({ scopeType, scopeId, amount, hardStopEnabled, isActive, unpricedUsagePolicy }) => ({ scopeType, scopeId, amount, hardStopEnabled, isActive, unpricedUsagePolicy }));
     expect(funding).toHaveLength(3);
     expect(funding.every(policy => policy.amount === CHAT_FEATURE_BUDGET_CENTS && policy.hardStopEnabled && policy.isActive && policy.unpricedUsagePolicy === "allow")).toBe(true);
-    await input.evidence("chat-feature-setup.json", { scenarioId: scenario.id, companyId: f.company.id,
+    const savedLead = await api.get<{ adapterConfig?: { timeoutSec?: unknown } }>(`/api/agents/${f.agent.id}`);
+    const nativeTimeoutPolicy = chatNativeTimeoutEvidence({ selectedTurnTimeoutMs: execution.task.turnTimeoutMs,
+      authoredLegacyTurnTimeoutMs: 120_000, attemptTimeoutMs: execution.task.attemptTimeoutMs[execution.environment.id], observationBudgetMs: 300_000,
+      savedAgentTimeoutSec: savedLead.adapterConfig?.timeoutSec });
+    await input.evidence("chat-feature-setup.json", { nativeTimeoutPolicy, scenarioId: scenario.id, companyId: f.company.id,
       leadId: f.agent.id, workerId, projectId, brief, budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS,
       maximumProviderRuns: execution.task.expectedRunCount, providerRunTimeoutMs: execution.task.turnTimeoutMs,
       setupObservations, funding, instructions: "Production role defaults; no fixture procedure or expected outcome is injected." });

tests/runner-e2e/chat-native-timeout-policy.test.ts

@@ -0,0 +1,49 @@
+import { expect, it } from "vitest";
+import { runnerMatrix, runnerSuites } from "./catalog.js";
+import { chatCatalogCases } from "./chat-catalog-registry.js";
+import { chatFeatureProfile } from "./chat-feature-cases.js";
+import { CHAT_NATIVE_TIMEOUT_POLICY, chatNativeTimeoutEvidence } from "./chat-native-timeout-policy.js";
+import type { RunnerProfileFixture } from "./types.js";
+
+it("uses the production native cap only in the catalog while preserving every finite scenario budget and run bound", () => {
+  const cells = runnerMatrix.filter(row => row.suite.id === "chat-features");
+  expect(cells).toHaveLength(244);
+  for (const cell of cells) {
+    expect(cell.task.turnTimeoutMs).toBe(0);
+    const declaration = chatCatalogCases.find(row => row.scenarioId === cell.task.id);
+    if (declaration) {
+      expect(cell.task.attemptTimeoutMs).toEqual({ local: declaration.attemptTimeoutMs, daytona: declaration.attemptTimeoutMs });
+      expect(cell.task.expectedRunCount).toBe(declaration.expectedRunCount);
+      expect(cell.task.minimumExpectedRunCount).toBe(declaration.minimumRunCount);
+      expect(declaration.turnTimeoutMs).toBeGreaterThan(0);
+    }
+    expect(cell.task.automaticRetryPolicy).toBe("single_attempt");
+  }
+  const byId = (id: string) => cells.find(row => row.task.id === id)!;
+  expect(byId("projects.create-project").task).toMatchObject({ expectedRunCount: 2, attemptTimeoutMs: { local: 420_000 } });
+  expect(byId("projects.project-work").task).toMatchObject({ expectedRunCount: 3, minimumExpectedRunCount: 1, attemptTimeoutMs: { local: 300_000 } });
+  expect(byId("tasks.organize-open-work").task.attemptTimeoutMs.local).toBe(600_000);
+  expect(runnerSuites.find(row => row.id === "chat-features")?.definitionMetadata).toMatchObject({ nativeTimeoutPolicy: CHAT_NATIVE_TIMEOUT_POLICY });
+  expect(runnerMatrix.filter(row => row.suite.id !== "chat-features").some(row => (row.task.turnTimeoutMs ?? 0) > 0)).toBe(true);
+});
+
+it("saves the actual catalog profile with no native cap without removing funding, daily admission or unrelated adapter settings", () => {
+  const original = { adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", timeoutSec: 120, retained: true }, runtimeConfig: { retained: true } };
+  const profile = { provider: "opencode", buildAgent: () => structuredClone(original) } as unknown as RunnerProfileFixture;
+  const input = {} as Parameters<RunnerProfileFixture["buildAgent"]>[0];
+  const saved = chatFeatureProfile(profile).buildAgent(input);
+  expect(saved.adapterConfig).toEqual({ ...original.adapterConfig, timeoutSec: 0 });
+  expect(saved.budgetMonthlyCents).toBe(500);
+  expect(saved.runtimeConfig).toEqual({ retained: true, heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: 500 } });
+  expect(profile.buildAgent(input)).toEqual(original);
+});
+
+it("reports zero as no per-turn cap alongside real finite lifetime and rejects mismatched saved or selected settings", () => {
+  const input = { selectedTurnTimeoutMs: 0, savedAgentTimeoutSec: 0, authoredLegacyTurnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, observationBudgetMs: 270_000 };
+  expect(chatNativeTimeoutEvidence(input)).toMatchObject({ effectiveNativeTurnTimeoutMs: 0,
+    scenarioCellBudgetMs: 300_000, scenarioObservationBudgetMs: 270_000, playwrightBudgetMs: 390_000,
+    launcherWatchdogMs: 690_000, activeRunCancellationWaitMs: 45_000, ownedProcessShutdownGraceMs: 45_000, ownedProcessKillGraceMs: 5_000 });
+  for (const change of [{ selectedTurnTimeoutMs: 120_000 }, { savedAgentTimeoutSec: 120 }, { savedAgentTimeoutSec: undefined },
+    { observationBudgetMs: Infinity }, { observationBudgetMs: 0 }, { observationBudgetMs: 300_000 }, { attemptTimeoutMs: 0 }, { attemptTimeoutMs: Infinity }, { authoredLegacyTurnTimeoutMs: 0 }])
+    expect(() => chatNativeTimeoutEvidence({ ...input, ...change })).toThrow();
+});

tests/runner-e2e/chat-native-timeout-policy.ts

@@ -0,0 +1,32 @@
+export const CHAT_NATIVE_TIMEOUT_POLICY = {
+  kind: "production-default-no-active-turn-cap",
+  effectiveNativeTurnTimeoutMs: 0,
+  savedAgentTimeoutSec: 0,
+  lifetimeOwner: "unchanged-finite-scenario-cell-and-owned-cleanup",
+} as const;
+
+export function chatNativeTimeoutEvidence(input: {
+  selectedTurnTimeoutMs: number | undefined;
+  authoredLegacyTurnTimeoutMs: number;
+  attemptTimeoutMs: number;
+  observationBudgetMs: number;
+  savedAgentTimeoutSec: unknown;
+}) {
+  if (input.selectedTurnTimeoutMs !== CHAT_NATIVE_TIMEOUT_POLICY.effectiveNativeTurnTimeoutMs
+    || input.savedAgentTimeoutSec !== CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec)
+    throw new Error("Catalog effective native timeout differs from the production-default policy");
+  if (!Number.isFinite(input.attemptTimeoutMs) || input.attemptTimeoutMs <= 30_000
+    || !Number.isFinite(input.observationBudgetMs) || input.observationBudgetMs <= 0 || input.observationBudgetMs >= input.attemptTimeoutMs
+    || !Number.isFinite(input.authoredLegacyTurnTimeoutMs) || input.authoredLegacyTurnTimeoutMs <= 0)
+    throw new Error("Catalog requires its unchanged finite outer budget and explicit authored legacy declaration");
+  return { ...CHAT_NATIVE_TIMEOUT_POLICY,
+    authoredLegacyTurnTimeoutMs: input.authoredLegacyTurnTimeoutMs,
+    scenarioCellBudgetMs: input.attemptTimeoutMs,
+    scenarioObservationBudgetMs: input.observationBudgetMs,
+    playwrightBudgetMs: input.attemptTimeoutMs + 90_000,
+    launcherWatchdogMs: input.attemptTimeoutMs + 90_000 + 300_000,
+    activeRunCancellationWaitMs: 45_000,
+    ownedProcessShutdownGraceMs: 45_000,
+    ownedProcessKillGraceMs: 5_000,
+  };
+}