fix(evals): preserve original native observation clocks for eligible historical finals

This commit is contained in:
Dotta committed 2026-10-28 11:15:52 -05:00
1 parent 3e1c87117e
commit c455a206a8
15 files changed
+70 -26

No files matched your search

+19
View File
@@ -26,6 +26,25 @@ export interface ChatCatalogContext {
/** These exact selected OpenCode/local journeys use the original flow clock for
* native observation phases. Other cases/lanes keep their authored phase caps. */
const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.access.cli-authorization",
"chat-features.runner-opencode.local.access.other-company",
"chat-features.runner-opencode.local.access.private-resource",
"chat-features.runner-opencode.local.agent-chat.conference-room",
"chat-features.runner-opencode.local.agent-chat.track-delegated-work",
"chat-features.runner-opencode.local.agents.agent-api-key",
"chat-features.runner-opencode.local.agents.hire-needs-approval",
"chat-features.runner-opencode.local.agents.pause-agent",
"chat-features.runner-opencode.local.agents.retire-agent",
"chat-features.runner-opencode.local.app-permissions.new-quarantined-action",
"chat-features.runner-opencode.local.app-permissions.read-allowed-write-off",
"chat-features.runner-opencode.local.app-permissions.test-as-agent",
"chat-features.runner-opencode.local.assistant-connections.device-login",
"chat-features.runner-opencode.local.budgets-costs.explain-hard-stop",
"chat-features.runner-opencode.local.budgets-costs.set-budget",
"chat-features.runner-opencode.local.cases.case-history",
"chat-features.runner-opencode.local.inbox-search.archive-without-grant",
"chat-features.runner-opencode.local.inbox-search.show-my-work",
"chat-features.runner-opencode.local.tasks.ambiguous-task",
"chat-features.runner-opencode.local.secrets.secret-proposal",
"chat-features.runner-opencode.local.developer-labs.long-thread-performance",
"chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once",
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { randomBytes } from "node:crypto";
import { expect } from "@playwright/test";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -63,7 +64,7 @@ export const appTestExternalCases = [definition("app-permissions.test-as-agent",
}, gradeAppTest, {
authentication: "human-session", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/pages/apps/Browse.tsx", "../../ui/src/pages/apps/ConnectorsToolbar.tsx", "../../ui/src/pages/apps/connection-owner.tsx", "../../ui/src/pages/apps/AppDetail.tsx", "../../ui/src/pages/apps/app-detail/PermissionsPanel.tsx", "chat-scenarios/app-permissions.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length;
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const provider = providers.get(ctx)!; state.providerCallsBeforeBoard = provider.calls.filter(row => row.method === "tools/call").length;
const o = await observeExternal(ctx, state); if (!appTestGuidance(o.reply)) throw new Error("Chat must identify Morgan and the requested read action before the board test");
const prefix = String(state.connectionPath);
const navigation = appTestNavigation(o.reply, o.links, o.origin, prefix);
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { definition, externalObservation, observeExternal, checks, verifyRenderedReply } from "./external-support.js";
@@ -63,7 +64,7 @@ export const assistantDeviceExternalCases = [definition("assistant-connections.d
authentication: "human-session", attemptTimeoutMs: 780_000,
sourceFiles: ["chat-catalog-domains/external-assistant-device.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-device-private-contract.ts", "../../ui/src/pages/McpConnect.tsx", "../../server/src/routes/public-mcp.ts", "../../server/src/services/public-mcp/oauth.ts", "../../packages/shared/src/public-mcp.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Device fixture missing");
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Device fixture missing");
const approvalUi = await privateBrowserEntry({
mode: "device-approve", baseURL: origin, setupURL: fixture.primary.verification_uri_complete,
providerURL: origin, credential: fixture.primary.user_code, companyId: ctx.input.fixtures.company.id,
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import type { BudgetIncident, BudgetOverview } from "../../../packages/shared/src/types/budget.js";
import type { CostEvent } from "../../../packages/shared/src/types/cost.js";
@@ -97,7 +98,7 @@ export const budgetIncidentIdentityCases: readonly ChatCatalogCase[] = ["budgets
&& incidentSourceFundingReady(budget, ctx.input.fixtures.company.id, ctx.input.fixtures.agent.id), "The real prerequisite budget incident and separate manual pause survive harness setup, without changing either hold."); return c.result();
},
async act(ctx, state) {
await waitForReply(ctx); if (scenarioId.endsWith("explain-hard-stop")) return;
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); if (scenarioId.endsWith("explain-hard-stop")) return;
const root = `/api/companies/${ctx.input.fixtures.company.id}`; const incident = state.incident as BudgetIncident;
const beforeHuman = await ctx.input.api.get<BudgetOverview>(`${root}/budgets/overview`);
const beforeHumanWorker = await ctx.input.api.get<PausedAgent>(`/api/agents/${entity(state, "worker").id}`);
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js";
import { checks, entity } from "./internal-support.js";
@@ -30,7 +31,7 @@ export const budgetIdentityCases = [identityDefinition("budgets-costs.set-budget
return { worker, other, workerConfig };
}, gradeBudget, {
async act(ctx, state) {
await waitForReply(ctx);
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx));
const root = `/api/companies/${ctx.input.fixtures.company.id}`; const worker = entity(state, "worker");
state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/budgets/policies`, "POST", {
scopeType: "agent", scopeId: worker.id, amount: 5000, windowKind: "calendar_month_utc" });
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { spawn, type ChildProcess } from "node:child_process";
import { appendFile, mkdir, readFile, rm } from "node:fs/promises";
import path from "node:path";
@@ -61,7 +62,7 @@ export const cliAuthorizationIdentityCases: readonly ChatCatalogCase[] = [identi
sourceFiles: ["chat-catalog-domains/identity-cli-authorization.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml",
"../../cli/src/index.ts", "../../cli/src/client/board-auth.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/commands/client/common.ts", "../../pnpm-lock.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing");
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing");
const login = runCli(ctx, ["auth", "login", "--company-id", ctx.input.fixtures.company.id, "--no-browser"], "login");
void login.catch(() => {}); // Later awaited; browser failures still trigger owned cleanup.
// Observe the CLI's own public challenge, never synthesize its success or
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { grantCaseTaskAssignment } from "./integrator-case-prerequisites.js";
import type { cases, caseEvents } from "../../../packages/db/src/schema/cases.js";
import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
@@ -69,9 +70,9 @@ export const integratorCaseHistoryCases = [integratorDefinition("cases", "cases.
workflow_document_text: `# Diagnosis review\nMorgan (${worker.id}) saves the initial event-handler binding hypothesis in the task-owned diagnosis document, then hands this same primary to Riley (${reviewer.id}). Riley reviews the actual synthetic browser evidence in the brief, revises that same diagnosis document to explain the touch overlay intercepting taps, updates the case diagnosis fact accordingly, then hands this same primary back to Morgan for final recording. Morgan's final-recording phase reads the preserved revisions, saves a task comment and completes this ordinary task without another handoff or case closure.`,
initialDocuments: [{ key: "brief", title: "Synthetic browser evidence", body: "Customer initially suspects an event-handler binding problem. Controlled reproduction later shows a transparent overlay intercepting touch on the Pay button; keyboard activation bypasses that hit target and succeeds. The overlay observation supersedes the preliminary event-binding hypothesis." }],
});
const result = await pollUntil({ label: "real diagnosis revisions and two distinct agent handoffs", deadlineAt: Date.now() + 540_000, intervalMs: 1000,
const result = await pollUntil({ label: "real diagnosis revisions and two distinct agent handoffs", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 540_000), intervalMs: 1000,
load: async () => ({ task: await ctx.input.api.get<Issue>(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns(), handoffs: (await ctx.input.api.get<{ items: CaseHandoff[] }>(`/api/cases/${kickoff.caseId}/handoffs?limit=100`)).items }),
accept: v => v.task.status === "done" && v.runs.length === 3 && v.runs.every(run => run.status === "succeeded") && v.handoffs.length === 3,
accept: v => chatCatalogNativePhaseIsTimely(ctx) && v.task.status === "done" && v.runs.length === 3 && v.runs.every(run => run.status === "succeeded") && v.handoffs.length === 3,
reject: v => v.runs.length > 3 || v.handoffs.length > 3 ? "Diagnosis prerequisite exceeded declared execution/handoff bounds" : undefined });
return { project: p, worker: { id: worker.id }, reviewer: { id: reviewer.id }, task: result.task, case: await ctx.input.api.get<CaseRecord>(`/api/cases/${kickoff.caseId}`) };
} catch (error) { await cleanup(ctx); throw error; }
@@ -82,8 +83,8 @@ export const integratorCaseHistoryCases = [integratorDefinition("cases", "cases.
}, { expectedRunCount: 5, expectedHumanMessageCount: 2, attemptTimeoutMs: 1_200_000, cleanup,
sourceFiles: ["chat-catalog-domains/integrator-case-history.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"],
async act(ctx, state) {
const first = await pollUntil({ label: "canonical diagnosis history answer before distinct missing-history question", deadlineAt: Date.now() + 180_000, intervalMs: 1000,
load: () => observeHistory(ctx, state), accept: o => o.runs.length === 4 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
const first = await pollUntil({ label: "canonical diagnosis history answer before distinct missing-history question", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000,
load: () => observeHistory(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.runs.length === 4 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
state.firstHistory = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-case-diagnosis-history-first.json", state.firstHistory);
const row = first.caseRecord; const beforeIds = new Set(first.caseEvents.map(event => event.id));
resources.get(ctx)!.unavailableText = `unretained-support-${ctx.input.nonce}`;
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { checks } from "./internal-support.js";
import { waitForReply, probeAgentPermission } from "./identity-support.js";
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
@@ -30,7 +31,7 @@ export const runtimeAgentKeyCases = [runtimeDefinition("agents.agent-api-key", a
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/runtime-agent-keys.ts", "chat-catalog-domains/runtime-private-key.ts", "chat-catalog-domains/runtime-private-key-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const worker = state.worker as RuntimeAgent;
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const worker = state.worker as RuntimeAgent;
const endpoint = `/api/agents/${worker.id}/keys`;
state.agentCreateStatus = await probeAgentPermission(ctx, endpoint, "POST", { name: "local integration" });
state.keysBeforeHuman = (await ctx.input.api.get<KeyRow[]>(endpoint)).length;
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { pollUntil } from "../api.js";
import { checks } from "./internal-support.js";
@@ -22,7 +23,7 @@ export function gradePauseAgent(value: unknown) {
c.add("exact-agent-identity", !!current && current.companyId === o.companyId, "The actual human control affects the same saved Morgan identity.");
return c.result();
}
export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", busyRuntimeWorker, gradePauseAgent, {
export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ctx => busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) }), gradePauseAgent, {
sourceFiles: ["chat-catalog-domains/runtime-agent-lifecycle.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
expectedRunCount: 2, attemptTimeoutMs: 600_000,
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] },
@@ -41,9 +42,9 @@ export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", bu
},
runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } },
async act(ctx, state) {
await pollUntil({ label: "Chat guidance while Morgan is still working", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Chat guidance while Morgan is still working", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
&& value.runs.find(run => run.id === state.workerRunId)?.status === "running" });
const worker = state.worker as RuntimeAgent;
state.beforeHumanStatus = (await ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`)).status;
@@ -66,13 +66,13 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options:
await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
return { worker, gate, previousRuns, company };
}
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent } = {}) {
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) {
const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true });
const task = await ctx.input.api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: Date.now() + 150_000,
const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: options.observationDeadlineAt ?? Date.now() + 150_000,
load: async () => ({ runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)), waiting: await readRuntimeTaskGate(ctx, gate, task.id, worker.id) }),
accept: value => value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running",
accept: value => (options.observationDeadlineAt === undefined || Date.now() < options.observationDeadlineAt) && value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running",
reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Busy fixture ended before its required active boundary" : undefined });
return { worker, task, gate, workerRunId: running.runs[0]!.id, workerWasReallyWaiting: true };
}
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { checks } from "./internal-support.js";
import { waitForReply } from "./identity-support.js";
@@ -26,7 +27,7 @@ export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference-
}, gradeConference, {
sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"],
async act(ctx, state) {
await waitForReply(ctx); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id;
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id;
const o = await observeRuntime(ctx, state);
const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL);
return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; });
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import { pollUntil } from "../api.js";
@@ -41,7 +42,7 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate
await ctx.input.api.saveAgentInstructions(prepared.worker.id, instructions);
const prompt = `Create one task titled Website, assigned to Morgan, to draft our launch page. Morgan is waiting for the human brief, which will include the reference ${marker}. Let Morgan implement it on that separate work task. Track the result here and tell me here when the work is complete, with access to the actual result. You may begin that one handoff now; no additional task or project is needed.`;
await sendChatMessage(ctx.input.page, prompt);
const phase = await pollUntil({ label: "Actual conversation-created Morgan handoff", deadlineAt: Date.now() + 180_000,
const phase = await pollUntil({ label: "Actual conversation-created Morgan handoff", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => {
await ctx.refreshIssue();
const tasks = await ctx.input.api.get<Entity[]>(`${prepared.company}/issues?includeAgentCreatedTasks=true&limit=1000`);
@@ -49,7 +50,7 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate
return { source: ctx.issue(), tasks, runs: await ctx.allRuns(),
waiting: candidates.length === 1 ? await readRuntimeTaskGate(ctx, prepared.gate, candidates[0]!.id, prepared.worker.id) : "" };
},
accept: value => value.source?.conversationState === "waiting" && value.waiting === "waiting" && value.runs.length === 2
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.waiting === "waiting" && value.runs.length === 2
&& value.runs.some(run => run.agentId === prepared.worker.id && run.status === "running")
&& value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
&& value.tasks.filter(task => task.id !== source.id).length === 1,
@@ -79,9 +80,9 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate
return c.result();
},
async act(ctx, state) {
const phase = await pollUntil({ label: "Canonical handoff status response", deadlineAt: Date.now() + 150_000,
const phase = await pollUntil({ label: "Canonical handoff status response", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.length === 3
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 3
&& value.runs.find(run => run.id === state.workerRunId)?.status === "running"
&& value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded") });
const queryIds = phase.runs.filter(run => !(state.setupRunIds as string[]).includes(run.id)).map(run => run.id);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { pollUntil } from "../api.js";
import { sendChatMessage } from "../chat-flow.js";
@@ -7,11 +8,11 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js";
interface HireApproval { id: string; companyId: string; type: string; status: string; requestedByAgentId: string | null;
decidedByUserId: string | null; payload: { agentId?: string; name?: string; role?: string; title?: string; reportsTo?: string; adapterType?: string; adapterConfig?: unknown } }
async function pendingHire(ctx: ChatCatalogContext, name: string) {
return pollUntil({ label: `Actual governed ${name} proposal`, deadlineAt: Date.now() + 180_000,
return pollUntil({ label: `Actual governed ${name} proposal`, deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(),
approvals: await ctx.input.api.get<HireApproval[]>(`/api/companies/${ctx.input.fixtures.company.id}/approvals`),
agents: await ctx.input.api.get<RuntimeAgent[]>(`/api/companies/${ctx.input.fixtures.company.id}/agents`) }; },
accept: value => value.source?.conversationState === "waiting" && value.approvals.filter(row => row.type === "hire_agent" && row.status === "pending" && row.payload.name === name).length === 1
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.approvals.filter(row => row.type === "hire_agent" && row.status === "pending" && row.payload.name === name).length === 1
&& value.agents.some(row => row.name === name && row.status === "pending_approval")
&& value.runs.every(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"),
reject: value => value.runs.length > 4 || value.runs.some(run => run.agentId !== ctx.input.fixtures.agent.id || ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Governed hire admitted unexpected work or failed" : undefined });
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { pollUntil } from "../api.js";
import { checks, sameTask, type Entity } from "./internal-support.js";
@@ -27,7 +28,7 @@ export function gradeRetirement(value: unknown) {
return c.result();
}
export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", async ctx => {
const state = await busyRuntimeWorker(ctx);
const state = await busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) });
// Truthful historical fixture: this was completed by the human and assigned
// to Morgan for recordkeeping. It does not impersonate prior provider work.
const historicalTask = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, {
@@ -59,9 +60,9 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent",
return c.result();
},
async act(ctx, state) {
await pollUntil({ label: "Canonical retirement guidance while Morgan works", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Canonical retirement guidance while Morgan works", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 2
&& value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
&& value.runs.find(run => run.id === state.workerRunId)?.status === "running" });
const o = await observeRuntime(ctx, state); state.queryReply = o.reply;
@@ -0,0 +1,13 @@
import {afterEach,expect,it,vi} from "vitest";
import type {ChatCatalogContext} from "./chat-catalog-contract.js";
import {chatCatalogNativePhaseDeadline,chatCatalogNativePhaseIsTimely,chatCatalogOriginalPhaseDeadline} from "./chat-catalog-contract.js";
import {pollUntil} from "./api.js";
void pollUntil;
import {waitForReply} from "./chat-catalog-domains/identity-support.js";
const ids=["access.cli-authorization", "access.other-company", "access.private-resource", "agent-chat.conference-room", "agent-chat.track-delegated-work", "agents.agent-api-key", "agents.hire-needs-approval", "agents.pause-agent", "agents.retire-agent", "app-permissions.new-quarantined-action", "app-permissions.read-allowed-write-off", "app-permissions.test-as-agent", "assistant-connections.device-login", "budgets-costs.explain-hard-stop", "budgets-costs.set-budget", "cases.case-history", "inbox-search.archive-without-grant", "inbox-search.show-my-work", "tasks.ambiguous-task"];
const ctx=(id:string,deadline=270000)=>({scenario:{id},input:{execution:{id:`chat-features.runner-opencode.local.${id}`}},observationDeadlineAt:deadline,issue:()=>({conversationState:"waiting"}),refreshIssue:async()=>{},allRuns:async()=>[{status:Date.now()>=180000?"succeeded":"running"}]} as unknown as ChatCatalogContext);
afterEach(()=>vi.useRealTimers());
it.each(ids)("%s consumes the original setup-inclusive clock with no new grace",id=>{vi.useFakeTimers();vi.setSystemTime(200000);const c=ctx(id);expect(chatCatalogNativePhaseDeadline(c,350000)).toBe(270000);expect(chatCatalogOriginalPhaseDeadline(c)).toBe(270000);vi.setSystemTime(270000);expect(chatCatalogNativePhaseIsTimely(c)).toBe(false);expect(()=>chatCatalogOriginalPhaseDeadline(c)).toThrow(/exhausted/);});
it("actual selected identity reply wait survives its artificial150s cap but keeps the original deadline",async()=>{vi.useFakeTimers();vi.setSystemTime(0);const c=ctx("budgets-costs.set-budget");const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(0);await vi.advanceTimersByTimeAsync(181000);expect(await result).toBe("accepted");expect(c.observationDeadlineAt).toBe(270000);});
it("selected identity helper rejects a success read after the original clock",async()=>{vi.useFakeTimers();vi.setSystemTime(269999);const c=ctx("access.cli-authorization");c.refreshIssue=async()=>{vi.setSystemTime(270001);};c.allRuns=async()=>[{status:"succeeded"}] as never;const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(1000);expect(await result).toBe("expired");});
it("keeps exhausted cases, provenance controls and other provider lanes outside this new clock scope",()=>{vi.useFakeTimers();vi.setSystemTime(200000);for(const id of ["chat-channels.provider-disabled","access.sign-in","agents.hire-denied","agent-chat.chat-disabled","tasks.create-backlog","activity.audit-change","cli-operations.choose-cli-context","companies.rename-company"]){const c=ctx(id);expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();expect(chatCatalogNativePhaseDeadline(c,350000)).toBe(350000);}const c=ctx(ids[0]!);Object.assign(c.input.execution,{id:`chat-features.runner-codex.local.${ids[0]}`});expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();});