From c455a206a8de97def92c2a95d5f27c0fd046987f Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 11:15:52 -0500 Subject: [PATCH] fix(evals): preserve original native observation clocks for eligible historical finals --- tests/runner-e2e/chat-catalog-contract.ts | 19 +++++++++++++++++++ .../chat-catalog-domains/external-app-test.ts | 3 ++- .../external-assistant-device.ts | 3 ++- .../identity-budget-incidents.ts | 3 ++- .../identity-budgets-costs.ts | 3 ++- .../identity-cli-authorization.ts | 3 ++- .../integrator-case-history.ts | 9 +++++---- .../runtime-agent-keys.ts | 3 ++- .../runtime-agent-lifecycle.ts | 7 ++++--- .../runtime-busy-worker.ts | 6 +++--- .../runtime-conference.ts | 3 ++- .../chat-catalog-domains/runtime-handoffs.ts | 9 +++++---- .../chat-catalog-domains/runtime-hiring.ts | 5 +++-- .../runtime-retirement.ts | 7 ++++--- .../chat-historical-native-clock.test.ts | 13 +++++++++++++ 15 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 tests/runner-e2e/chat-historical-native-clock.test.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 64d8efd5976..7a421ffe494 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -26,6 +26,25 @@ export interface ChatCatalogContext { /** These exact selected OpenCode/local journeys use the original flow clock for * native observation phases. Other cases/lanes keep their authored phase caps. */ const originalClockExecutionIds = new Set([ + "chat-features.runner-opencode.local.access.cli-authorization", + "chat-features.runner-opencode.local.access.other-company", + "chat-features.runner-opencode.local.access.private-resource", + "chat-features.runner-opencode.local.agent-chat.conference-room", + "chat-features.runner-opencode.local.agent-chat.track-delegated-work", + "chat-features.runner-opencode.local.agents.agent-api-key", + "chat-features.runner-opencode.local.agents.hire-needs-approval", + "chat-features.runner-opencode.local.agents.pause-agent", + "chat-features.runner-opencode.local.agents.retire-agent", + "chat-features.runner-opencode.local.app-permissions.new-quarantined-action", + "chat-features.runner-opencode.local.app-permissions.read-allowed-write-off", + "chat-features.runner-opencode.local.app-permissions.test-as-agent", + "chat-features.runner-opencode.local.assistant-connections.device-login", + "chat-features.runner-opencode.local.budgets-costs.explain-hard-stop", + "chat-features.runner-opencode.local.budgets-costs.set-budget", + "chat-features.runner-opencode.local.cases.case-history", + "chat-features.runner-opencode.local.inbox-search.archive-without-grant", + "chat-features.runner-opencode.local.inbox-search.show-my-work", + "chat-features.runner-opencode.local.tasks.ambiguous-task", "chat-features.runner-opencode.local.secrets.secret-proposal", "chat-features.runner-opencode.local.developer-labs.long-thread-performance", "chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once", diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts index 327b1bd224e..a1ea96f087a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { randomBytes } from "node:crypto"; import { expect } from "@playwright/test"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; @@ -63,7 +64,7 @@ export const appTestExternalCases = [definition("app-permissions.test-as-agent", }, gradeAppTest, { authentication: "human-session", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/pages/apps/Browse.tsx", "../../ui/src/pages/apps/ConnectorsToolbar.tsx", "../../ui/src/pages/apps/connection-owner.tsx", "../../ui/src/pages/apps/AppDetail.tsx", "../../ui/src/pages/apps/app-detail/PermissionsPanel.tsx", "chat-scenarios/app-permissions.yaml"], async act(ctx, state) { - await waitForReply(ctx); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length; + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length; const o = await observeExternal(ctx, state); if (!appTestGuidance(o.reply)) throw new Error("Chat must identify Morgan and the requested read action before the board test"); const prefix = String(state.connectionPath); const navigation = appTestNavigation(o.reply, o.links, o.origin, prefix); diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-device.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-device.ts index 54b2482a98f..20fe16a13f2 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-assistant-device.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-device.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { definition, externalObservation, observeExternal, checks, verifyRenderedReply } from "./external-support.js"; @@ -63,7 +64,7 @@ export const assistantDeviceExternalCases = [definition("assistant-connections.d authentication: "human-session", attemptTimeoutMs: 780_000, sourceFiles: ["chat-catalog-domains/external-assistant-device.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-device-private-contract.ts", "../../ui/src/pages/McpConnect.tsx", "../../server/src/routes/public-mcp.ts", "../../server/src/services/public-mcp/oauth.ts", "../../packages/shared/src/public-mcp.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"], async act(ctx, state) { - await waitForReply(ctx); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Device fixture missing"); + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Device fixture missing"); const approvalUi = await privateBrowserEntry({ mode: "device-approve", baseURL: origin, setupURL: fixture.primary.verification_uri_complete, providerURL: origin, credential: fixture.primary.user_code, companyId: ctx.input.fixtures.company.id, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts index 929307ef200..2589e0af94b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budget-incidents.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import type { BudgetIncident, BudgetOverview } from "../../../packages/shared/src/types/budget.js"; import type { CostEvent } from "../../../packages/shared/src/types/cost.js"; @@ -97,7 +98,7 @@ export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets && incidentSourceFundingReady(budget, ctx.input.fixtures.company.id, ctx.input.fixtures.agent.id), "The real prerequisite budget incident and separate manual pause survive harness setup, without changing either hold."); return c.result(); }, async act(ctx, state) { - await waitForReply(ctx); if (scenarioId.endsWith("explain-hard-stop")) return; + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); if (scenarioId.endsWith("explain-hard-stop")) return; const root = `/api/companies/${ctx.input.fixtures.company.id}`; const incident = state.incident as BudgetIncident; const beforeHuman = await ctx.input.api.get(`${root}/budgets/overview`); const beforeHumanWorker = await ctx.input.api.get(`/api/agents/${entity(state, "worker").id}`); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts b/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts index badd85e8cd3..891aae1f347 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js"; import { checks, entity } from "./internal-support.js"; @@ -30,7 +31,7 @@ export const budgetIdentityCases = [identityDefinition("budgets-costs.set-budget return { worker, other, workerConfig }; }, gradeBudget, { async act(ctx, state) { - await waitForReply(ctx); + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const worker = entity(state, "worker"); state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/budgets/policies`, "POST", { scopeType: "agent", scopeId: worker.id, amount: 5000, windowKind: "calendar_month_utc" }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts index 439b9ece734..3af6256d802 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { spawn, type ChildProcess } from "node:child_process"; import { appendFile, mkdir, readFile, rm } from "node:fs/promises"; import path from "node:path"; @@ -61,7 +62,7 @@ export const cliAuthorizationIdentityCases: readonly ChatCatalogCase[] = [identi sourceFiles: ["chat-catalog-domains/identity-cli-authorization.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml", "../../cli/src/index.ts", "../../cli/src/client/board-auth.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/commands/client/common.ts", "../../pnpm-lock.yaml"], async act(ctx, state) { - await waitForReply(ctx); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing"); + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing"); const login = runCli(ctx, ["auth", "login", "--company-id", ctx.input.fixtures.company.id, "--no-browser"], "login"); void login.catch(() => {}); // Later awaited; browser failures still trigger owned cleanup. // Observe the CLI's own public challenge, never synthesize its success or diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-history.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-history.ts index a00d6adc96e..451b63118c8 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-history.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { grantCaseTaskAssignment } from "./integrator-case-prerequisites.js"; import type { cases, caseEvents } from "../../../packages/db/src/schema/cases.js"; import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js"; @@ -69,9 +70,9 @@ export const integratorCaseHistoryCases = [integratorDefinition("cases", "cases. workflow_document_text: `# Diagnosis review\nMorgan (${worker.id}) saves the initial event-handler binding hypothesis in the task-owned diagnosis document, then hands this same primary to Riley (${reviewer.id}). Riley reviews the actual synthetic browser evidence in the brief, revises that same diagnosis document to explain the touch overlay intercepting taps, updates the case diagnosis fact accordingly, then hands this same primary back to Morgan for final recording. Morgan's final-recording phase reads the preserved revisions, saves a task comment and completes this ordinary task without another handoff or case closure.`, initialDocuments: [{ key: "brief", title: "Synthetic browser evidence", body: "Customer initially suspects an event-handler binding problem. Controlled reproduction later shows a transparent overlay intercepting touch on the Pay button; keyboard activation bypasses that hit target and succeeds. The overlay observation supersedes the preliminary event-binding hypothesis." }], }); - const result = await pollUntil({ label: "real diagnosis revisions and two distinct agent handoffs", deadlineAt: Date.now() + 540_000, intervalMs: 1000, + const result = await pollUntil({ label: "real diagnosis revisions and two distinct agent handoffs", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 540_000), intervalMs: 1000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns(), handoffs: (await ctx.input.api.get<{ items: CaseHandoff[] }>(`/api/cases/${kickoff.caseId}/handoffs?limit=100`)).items }), - accept: v => v.task.status === "done" && v.runs.length === 3 && v.runs.every(run => run.status === "succeeded") && v.handoffs.length === 3, + accept: v => chatCatalogNativePhaseIsTimely(ctx) && v.task.status === "done" && v.runs.length === 3 && v.runs.every(run => run.status === "succeeded") && v.handoffs.length === 3, reject: v => v.runs.length > 3 || v.handoffs.length > 3 ? "Diagnosis prerequisite exceeded declared execution/handoff bounds" : undefined }); return { project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, task: result.task, case: await ctx.input.api.get(`/api/cases/${kickoff.caseId}`) }; } catch (error) { await cleanup(ctx); throw error; } @@ -82,8 +83,8 @@ export const integratorCaseHistoryCases = [integratorDefinition("cases", "cases. }, { expectedRunCount: 5, expectedHumanMessageCount: 2, attemptTimeoutMs: 1_200_000, cleanup, sourceFiles: ["chat-catalog-domains/integrator-case-history.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], async act(ctx, state) { - const first = await pollUntil({ label: "canonical diagnosis history answer before distinct missing-history question", deadlineAt: Date.now() + 180_000, intervalMs: 1000, - load: () => observeHistory(ctx, state), accept: o => o.runs.length === 4 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const first = await pollUntil({ label: "canonical diagnosis history answer before distinct missing-history question", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000, + load: () => observeHistory(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.runs.length === 4 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); state.firstHistory = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-case-diagnosis-history-first.json", state.firstHistory); const row = first.caseRecord; const beforeIds = new Set(first.caseEvents.map(event => event.id)); resources.get(ctx)!.unavailableText = `unretained-support-${ctx.input.nonce}`; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts index 66d1ab3f4a0..713c0f82255 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { checks } from "./internal-support.js"; import { waitForReply, probeAgentPermission } from "./identity-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; @@ -30,7 +31,7 @@ export const runtimeAgentKeyCases = [runtimeDefinition("agents.agent-api-key", a authentication: "human-session", sourceFiles: ["chat-catalog-domains/runtime-agent-keys.ts", "chat-catalog-domains/runtime-private-key.ts", "chat-catalog-domains/runtime-private-key-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { - await waitForReply(ctx); const worker = state.worker as RuntimeAgent; + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const worker = state.worker as RuntimeAgent; const endpoint = `/api/agents/${worker.id}/keys`; state.agentCreateStatus = await probeAgentPermission(ctx, endpoint, "POST", { name: "local integration" }); state.keysBeforeHuman = (await ctx.input.api.get(endpoint)).length; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts index 3f5f438cbed..a1e3237af34 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { checks } from "./internal-support.js"; @@ -22,7 +23,7 @@ export function gradePauseAgent(value: unknown) { c.add("exact-agent-identity", !!current && current.companyId === o.companyId, "The actual human control affects the same saved Morgan identity."); return c.result(); } -export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", busyRuntimeWorker, gradePauseAgent, { +export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ctx => busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) }), gradePauseAgent, { sourceFiles: ["chat-catalog-domains/runtime-agent-lifecycle.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], expectedRunCount: 2, attemptTimeoutMs: 600_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, @@ -41,9 +42,9 @@ export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", bu }, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } }, async act(ctx, state) { - await pollUntil({ label: "Chat guidance while Morgan is still working", deadlineAt: Date.now() + 150_000, + await pollUntil({ label: "Chat guidance while Morgan is still working", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") && value.runs.find(run => run.id === state.workerRunId)?.status === "running" }); const worker = state.worker as RuntimeAgent; state.beforeHumanStatus = (await ctx.input.api.get(`/api/agents/${worker.id}`)).status; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts index cde8cbfb3a2..3b8bdcd1166 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -66,13 +66,13 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); return { worker, gate, previousRuns, company }; } -export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent } = {}) { +export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) { const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true }); const task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); - const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: Date.now() + 150_000, + const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: options.observationDeadlineAt ?? Date.now() + 150_000, load: async () => ({ runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)), waiting: await readRuntimeTaskGate(ctx, gate, task.id, worker.id) }), - accept: value => value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running", + accept: value => (options.observationDeadlineAt === undefined || Date.now() < options.observationDeadlineAt) && value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running", reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Busy fixture ended before its required active boundary" : undefined }); return { worker, task, gate, workerRunId: running.runs[0]!.id, workerWasReallyWaiting: true }; } diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts index 7db9f1fef1e..17f3c59af89 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { checks } from "./internal-support.js"; import { waitForReply } from "./identity-support.js"; @@ -26,7 +27,7 @@ export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference- }, gradeConference, { sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"], async act(ctx, state) { - await waitForReply(ctx); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id; + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id; const o = await observeRuntime(ctx, state); const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts index 8c683dfd3b0..b01f291db6e 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { writeFile } from "node:fs/promises"; import { pollUntil } from "../api.js"; @@ -41,7 +42,7 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate await ctx.input.api.saveAgentInstructions(prepared.worker.id, instructions); const prompt = `Create one task titled Website, assigned to Morgan, to draft our launch page. Morgan is waiting for the human brief, which will include the reference ${marker}. Let Morgan implement it on that separate work task. Track the result here and tell me here when the work is complete, with access to the actual result. You may begin that one handoff now; no additional task or project is needed.`; await sendChatMessage(ctx.input.page, prompt); - const phase = await pollUntil({ label: "Actual conversation-created Morgan handoff", deadlineAt: Date.now() + 180_000, + const phase = await pollUntil({ label: "Actual conversation-created Morgan handoff", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), load: async () => { await ctx.refreshIssue(); const tasks = await ctx.input.api.get(`${prepared.company}/issues?includeAgentCreatedTasks=true&limit=1000`); @@ -49,7 +50,7 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate return { source: ctx.issue(), tasks, runs: await ctx.allRuns(), waiting: candidates.length === 1 ? await readRuntimeTaskGate(ctx, prepared.gate, candidates[0]!.id, prepared.worker.id) : "" }; }, - accept: value => value.source?.conversationState === "waiting" && value.waiting === "waiting" && value.runs.length === 2 + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.waiting === "waiting" && value.runs.length === 2 && value.runs.some(run => run.agentId === prepared.worker.id && run.status === "running") && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") && value.tasks.filter(task => task.id !== source.id).length === 1, @@ -79,9 +80,9 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate return c.result(); }, async act(ctx, state) { - const phase = await pollUntil({ label: "Canonical handoff status response", deadlineAt: Date.now() + 150_000, + const phase = await pollUntil({ label: "Canonical handoff status response", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => value.source?.conversationState === "waiting" && value.runs.length === 3 + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 3 && value.runs.find(run => run.id === state.workerRunId)?.status === "running" && value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded") }); const queryIds = phase.runs.filter(run => !(state.setupRunIds as string[]).includes(run.id)).map(run => run.id); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-hiring.ts b/tests/runner-e2e/chat-catalog-domains/runtime-hiring.ts index e701d6cd54f..578e3823cc6 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-hiring.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-hiring.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; @@ -7,11 +8,11 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface HireApproval { id: string; companyId: string; type: string; status: string; requestedByAgentId: string | null; decidedByUserId: string | null; payload: { agentId?: string; name?: string; role?: string; title?: string; reportsTo?: string; adapterType?: string; adapterConfig?: unknown } } async function pendingHire(ctx: ChatCatalogContext, name: string) { - return pollUntil({ label: `Actual governed ${name} proposal`, deadlineAt: Date.now() + 180_000, + return pollUntil({ label: `Actual governed ${name} proposal`, deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), approvals: await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/approvals`), agents: await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/agents`) }; }, - accept: value => value.source?.conversationState === "waiting" && value.approvals.filter(row => row.type === "hire_agent" && row.status === "pending" && row.payload.name === name).length === 1 + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.approvals.filter(row => row.type === "hire_agent" && row.status === "pending" && row.payload.name === name).length === 1 && value.agents.some(row => row.name === name && row.status === "pending_approval") && value.runs.every(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"), reject: value => value.runs.length > 4 || value.runs.some(run => run.agentId !== ctx.input.fixtures.agent.id || ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Governed hire admitted unexpected work or failed" : undefined }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts index c2f4a300672..213778777f8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; @@ -27,7 +28,7 @@ export function gradeRetirement(value: unknown) { return c.result(); } export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", async ctx => { - const state = await busyRuntimeWorker(ctx); + const state = await busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) }); // Truthful historical fixture: this was completed by the human and assigned // to Morgan for recordkeeping. It does not impersonate prior provider work. const historicalTask = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { @@ -59,9 +60,9 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", return c.result(); }, async act(ctx, state) { - await pollUntil({ label: "Canonical retirement guidance while Morgan works", deadlineAt: Date.now() + 150_000, + await pollUntil({ label: "Canonical retirement guidance while Morgan works", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") && value.runs.find(run => run.id === state.workerRunId)?.status === "running" }); const o = await observeRuntime(ctx, state); state.queryReply = o.reply; diff --git a/tests/runner-e2e/chat-historical-native-clock.test.ts b/tests/runner-e2e/chat-historical-native-clock.test.ts new file mode 100644 index 00000000000..eff4fc9c99a --- /dev/null +++ b/tests/runner-e2e/chat-historical-native-clock.test.ts @@ -0,0 +1,13 @@ +import {afterEach,expect,it,vi} from "vitest"; +import type {ChatCatalogContext} from "./chat-catalog-contract.js"; +import {chatCatalogNativePhaseDeadline,chatCatalogNativePhaseIsTimely,chatCatalogOriginalPhaseDeadline} from "./chat-catalog-contract.js"; +import {pollUntil} from "./api.js"; +void pollUntil; +import {waitForReply} from "./chat-catalog-domains/identity-support.js"; +const ids=["access.cli-authorization", "access.other-company", "access.private-resource", "agent-chat.conference-room", "agent-chat.track-delegated-work", "agents.agent-api-key", "agents.hire-needs-approval", "agents.pause-agent", "agents.retire-agent", "app-permissions.new-quarantined-action", "app-permissions.read-allowed-write-off", "app-permissions.test-as-agent", "assistant-connections.device-login", "budgets-costs.explain-hard-stop", "budgets-costs.set-budget", "cases.case-history", "inbox-search.archive-without-grant", "inbox-search.show-my-work", "tasks.ambiguous-task"]; +const ctx=(id:string,deadline=270000)=>({scenario:{id},input:{execution:{id:`chat-features.runner-opencode.local.${id}`}},observationDeadlineAt:deadline,issue:()=>({conversationState:"waiting"}),refreshIssue:async()=>{},allRuns:async()=>[{status:Date.now()>=180000?"succeeded":"running"}]} as unknown as ChatCatalogContext); +afterEach(()=>vi.useRealTimers()); +it.each(ids)("%s consumes the original setup-inclusive clock with no new grace",id=>{vi.useFakeTimers();vi.setSystemTime(200000);const c=ctx(id);expect(chatCatalogNativePhaseDeadline(c,350000)).toBe(270000);expect(chatCatalogOriginalPhaseDeadline(c)).toBe(270000);vi.setSystemTime(270000);expect(chatCatalogNativePhaseIsTimely(c)).toBe(false);expect(()=>chatCatalogOriginalPhaseDeadline(c)).toThrow(/exhausted/);}); +it("actual selected identity reply wait survives its artificial150s cap but keeps the original deadline",async()=>{vi.useFakeTimers();vi.setSystemTime(0);const c=ctx("budgets-costs.set-budget");const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(0);await vi.advanceTimersByTimeAsync(181000);expect(await result).toBe("accepted");expect(c.observationDeadlineAt).toBe(270000);}); +it("selected identity helper rejects a success read after the original clock",async()=>{vi.useFakeTimers();vi.setSystemTime(269999);const c=ctx("access.cli-authorization");c.refreshIssue=async()=>{vi.setSystemTime(270001);};c.allRuns=async()=>[{status:"succeeded"}] as never;const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(1000);expect(await result).toBe("expired");}); +it("keeps exhausted cases, provenance controls and other provider lanes outside this new clock scope",()=>{vi.useFakeTimers();vi.setSystemTime(200000);for(const id of ["chat-channels.provider-disabled","access.sign-in","agents.hire-denied","agent-chat.chat-disabled","tasks.create-backlog","activity.audit-change","cli-operations.choose-cli-context","companies.rename-company"]){const c=ctx(id);expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();expect(chatCatalogNativePhaseDeadline(c,350000)).toBe(350000);}const c=ctx(ids[0]!);Object.assign(c.input.execution,{id:`chat-features.runner-codex.local.${ids[0]}`});expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();});