fix(evals): share original observation clock across selected native phases

This commit is contained in:
Dotta committed 2026-10-28 02:01:44 -05:00
1 parent 81abf04b21
commit d282ae2eee
19 files changed
+208 -73

No files matched your search

+48
View File
@@ -16,6 +16,54 @@ export interface ChatCatalogContext {
/** Available only to the two allowlisted native Slack ingress cases. */
selectSlackSource?(issueId: string): Promise<void>;
}
/** These exact selected OpenCode/local journeys use the original flow clock for
* native observation phases. Other cases/lanes keep their authored phase caps. */
const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.cli-operations.feedback-bundle",
"chat-features.runner-opencode.local.cli-operations.inspect-run-cli",
"chat-features.runner-opencode.local.cli-operations.owned-dev-instance",
"chat-features.runner-opencode.local.developer-labs.component-reference",
"chat-features.runner-opencode.local.developer-labs.gated-lab",
"chat-features.runner-opencode.local.developer-labs.interaction-fixture",
"chat-features.runner-opencode.local.recovery.browser-reconnect",
"chat-features.runner-opencode.local.recovery.controller-restart",
"chat-features.runner-opencode.local.recovery.explain-stall",
"chat-features.runner-opencode.local.recovery.retry-exhausted",
"chat-features.runner-opencode.local.recovery.source-action",
"chat-features.runner-opencode.local.routines.pause-owned",
"chat-features.runner-opencode.local.routines.run-now",
"chat-features.runner-opencode.local.steering.queue-follow-up",
"chat-features.runner-opencode.local.steering.stop-versus-pause",
"chat-features.runner-opencode.local.task-coordination.delegate-with-context",
"chat-features.runner-opencode.local.task-coordination.independent-review",
"chat-features.runner-opencode.local.task-coordination.pause-tree",
"chat-features.runner-opencode.local.tasks.edit-properties",
"chat-features.runner-opencode.local.steering.edit-queued-message",
"chat-features.runner-opencode.local.steering.discard-queued-message",
]);
function usesOriginalNativePhaseClock(ctx: ChatCatalogContext): boolean {
return originalClockExecutionIds.has(ctx.input?.execution?.id);
}
export function chatCatalogNativePhaseDeadline(ctx: ChatCatalogContext, legacyDeadlineAt: number): number {
if (!usesOriginalNativePhaseClock(ctx)) return legacyDeadlineAt;
const deadline = ctx.observationDeadlineAt, now = Date.now();
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original native observation budget exhausted or missing");
return deadline!;
}
/** Recheck the same clock before each declared human send operation. */
export function chatCatalogNativePhaseRemainingBudget(ctx: ChatCatalogContext): (() => number) | undefined {
if (!usesOriginalNativePhaseClock(ctx)) return undefined;
return () => {
const now = Date.now(), deadline = ctx.observationDeadlineAt;
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original native observation budget exhausted or missing");
return deadline! - now;
};
}
export function chatCatalogNativePhaseIsTimely(ctx: ChatCatalogContext): boolean {
return !usesOriginalNativePhaseClock(ctx) || Number.isFinite(ctx.observationDeadlineAt) && Date.now() < ctx.observationDeadlineAt!;
}
export type ChatCatalogTerminalRunStatus = "succeeded" | "cancelled" | "failed" | "timed_out";
export type ChatCatalogRunPolicy =
| { kind: "terminal-counts"; terminalStatusCounts: Partial<Record<ChatCatalogTerminalRunStatus, number>> }
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { createRoutineSchema } from "../../../packages/shared/src/validators/routine.js";
import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -67,8 +68,8 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines"
sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"],
expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
async act(ctx, state) {
const first = await pollUntil({ label: "routine invocation first actual reply", deadlineAt: Date.now() + 180_000, intervalMs: 1000,
load: () => observeInvocation(ctx, state), accept: o => o.invocations.length === 1 && o.runs.some(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id && row.status === "succeeded") && ctx.issue()?.conversationState === "waiting",
const first = await pollUntil({ label: "routine invocation first actual reply", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000,
load: () => observeInvocation(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.invocations.length === 1 && o.runs.some(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id && row.status === "succeeded") && ctx.issue()?.conversationState === "waiting"),
reject: o => o.invocations.length > 1 ? "Initial request created duplicate routine invocations" : undefined });
state.firstInvocation = { ...first, state: structuredClone(state) };
const invocation = first.invocations[0]!;
@@ -85,6 +86,6 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines"
load: () => observeInvocation(ctx, state), accept: o => o.invocations.length === 1 && o.invocations[0]?.status === "completed" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"),
reject: o => o.invocations.length > 1 || o.runs.length > 2 ? "Delivery replay admitted unexpected additional work" : undefined });
const { sendChatMessage } = await import("../chat-flow.js");
await sendChatMessage(ctx.input.page, "What was the actual result of that summary run?");
await sendChatMessage(ctx.input.page, "What was the actual result of that summary run?", chatCatalogNativePhaseRemainingBudget(ctx));
},
})];
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
@@ -75,8 +76,8 @@ export const integratorRoutinePauseCases = [integratorDefinition("routines", "ro
return c.result();
},
async act(ctx, state) {
const checkpoint = await pollUntil({ label: "actual pause reply while invocation remains running", deadlineAt: Date.now() + 150_000, intervalMs: 1000,
load: () => observePause(ctx, state), accept: o => o.runs.length === 3 && o.runs.some(run => run.agentId === o.leadId && run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
const checkpoint = await pollUntil({ label: "actual pause reply while invocation remains running", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), intervalMs: 1000,
load: () => observePause(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.runs.length === 3 && o.runs.some(run => run.agentId === o.leadId && run.status === "succeeded") && ctx.issue()?.conversationState === "waiting") });
state.pauseCheckpoint = { ...checkpoint, state: structuredClone(state) };
await ctx.input.evidence("chat-routine-pause-running-checkpoint.json", state.pauseCheckpoint);
gates.get(ctx)?.release();
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { collectRunEvents } from "../run-observations.js";
@@ -101,8 +102,8 @@ export function withEditOwnership(primary: ChatCatalogCase): ChatCatalogCase {
async setup(ctx) { const state = await primary.setup(ctx); const riley = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${riley.id}`, { name: "Riley" }); return { ...state, riley, canonicalEditQuery: ctx.scenario.query }; },
grade: value => gradeEditOwnership(value, primary.grade),
async act(ctx, state) {
const first = await pollUntil({ label: "Fully validated initial permitted property edit", deadlineAt: Date.now() + 150000,
load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" && primary.grade(o).passed });
const first = await pollUntil({ label: "Fully validated initial permitted property edit", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000),
load: () => observeInternal(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" && primary.grade(o).passed) });
state.firstEdit = { ...first, state: structuredClone(state) }; const riley = entity(state, "riley"); const t = entity(state, "task");
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: riley.id, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
gate = await startRunBriefGate(ctx, riley.id);
@@ -115,7 +116,7 @@ export function withEditOwnership(primary: ChatCatalogCase): ChatCatalogCase {
reject: value => value.runs.some(run => run.agentId === riley.id && ["failed", "cancelled", "succeeded", "timed_out"].includes(run.status)) ? "The conditional real checkout ended before the denial request" : undefined });
state.heldTask = held.task; state.heldRun = held.runs.find(run => run.agentId === riley.id); state.gateCalls = structuredClone(gate.calls);
await ctx.input.evidence("chat-catalog-edit-properties-boundary-variant.json", { firstEdit: state.firstEdit, heldTask: state.heldTask, heldRun: state.heldRun, gateCalls: state.gateCalls });
const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query);
const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query, chatCatalogNativePhaseRemainingBudget(ctx));
} catch (error) { await gate.close(); gate = undefined; throw error; }
},
async observe(ctx, state): Promise<EditOwnership> {
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { createHash } from "node:crypto";
import { readFile, realpath } from "node:fs/promises";
import path from "node:path";
@@ -83,9 +84,9 @@ const feedback = privilegedDefinition("cli-operations.feedback-bundle", "privile
sharingPreferenceBefore: sharingBefore.feedbackDataSharingPreference, workerWasReallyWaiting: true, cancelledBeforeCanonical: true, agentReportStatus, ...(await prepareOwnedCli(ctx)) };
},
async act(ctx, state) {
await pollUntil({ label: "Canonical human-export guidance settled", deadlineAt: Date.now() + 150000,
await pollUntil({ label: "Canonical human-export guidance settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000),
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.filter(r => r.status === "succeeded").length === 1 && o.runs.filter(r => r.status === "cancelled").length === 1 });
accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.filter(r => r.status === "succeeded").length === 1 && o.runs.filter(r => r.status === "cancelled").length === 1) });
const prior = await ctx.allRuns(); const company = ctx.input.fixtures.company.id, task = state.task as Entity;
const output = path.join(String(state.operatorDirectory), "feedback-export");
const result = JSON.parse(await runOwnedCli(ctx, ["feedback", "export", "--company-id", company, "--issue-id", task.id, "--out", output, "--json"])); state.cliExecuted = true;
@@ -129,14 +130,14 @@ const feedback = privilegedDefinition("cli-operations.feedback-bundle", "privile
remoteSharingEnabled: false, publicAttachmentCreated: false, outcomeKind: "operator interrupted unfinished work; not a model failure or failed-native run" };
await ctx.input.evidence("chat-catalog-local-feedback-export-receipt.json", receipt);
const message = `The human operator exported this local review bundle and retained the original interrupted work evidence. These are the observed facts; please report its location and contents without publishing it:\n\n${JSON.stringify(receipt)}`;
await sendChatMessage(ctx.input.page, message); state.exportReport = await companionPhaseForCancelled(ctx, prior.map(r => r.id), message);
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); state.exportReport = await companionPhaseForCancelled(ctx, prior.map(r => r.id), message);
}, grade: gradeFeedbackBundle, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
});
// companionPhase's default all-succeeded assertion intentionally does not apply to this retained cancelled fixture.
async function companionPhaseForCancelled(ctx: Parameters<typeof companionPhase>[0], prior: string[], message: string) {
const result = await pollUntil({ label: "Native reply to local interrupted-work export receipt", deadlineAt: Date.now() + 150000,
const result = await pollUntil({ label: "Native reply to local interrupted-work export receipt", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000),
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: o => o.issue?.conversationState === "waiting" && o.runs.length === prior.length + 1 && o.runs.filter(r => r.status === "succeeded").length === 2 && o.runs.filter(r => r.status === "cancelled").length === 1 });
accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.issue?.conversationState === "waiting" && o.runs.length === prior.length + 1 && o.runs.filter(r => r.status === "succeeded").length === 2 && o.runs.filter(r => r.status === "cancelled").length === 1) });
const run = result.runs.find(r => !prior.includes(r.id) && r.agentId === ctx.input.fixtures.agent.id)!;
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(c => !c.authorAgentId && c.body === message);
const replies = comments.filter(c => c.authorAgentId === ctx.input.fixtures.agent.id && c.createdByRunId === run.id);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { createHash } from "node:crypto";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -55,9 +56,9 @@ const inspect = privilegedDefinition("cli-operations.inspect-run-cli", "privileg
return { ...operator, ...busy, agentCancelStatus, initialRunReallyRunning: busy.workerWasReallyWaiting };
},
async act(ctx, state) {
const initial = await pollUntil({ label: "Canonical run-control guidance", deadlineAt: Date.now() + 150_000,
load: () => observeInternal(ctx, state), accept: o => !!o.reply && ctx.issue()?.conversationState === "waiting"
&& o.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded").length === 1 });
const initial = await pollUntil({ label: "Canonical run-control guidance", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: () => observeInternal(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (!!o.reply && ctx.issue()?.conversationState === "waiting"
&& o.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded").length === 1) });
const returned = inspectRunCommands(initial.reply, String(state.workerRunId));
await runOwnedCli(ctx, returned.commands[0]!);
const read = await ctx.input.api.get<{ id: string; agentId: string; companyId: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { observeInternal, observation, type Entity, type InternalObservation } from "./internal-support.js";
@@ -16,18 +17,18 @@ export function remainingPrivilegedObservationBudget(ctx: ChatCatalogContext): n
return deadline! - now;
}
export async function settlePrivilegedChat(ctx: ChatCatalogContext, observationDeadlineAt?: number): Promise<InternalObservation> {
return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000,
return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, observationDeadlineAt ?? Date.now() + 150_000),
load: () => observeInternal(ctx, {}),
accept: o => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting",
accept: o => chatCatalogNativePhaseIsTimely(ctx) && ((observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting"),
reject: o => o.runs.some(run => ["failed", "timed_out", "cancelled"].includes(run.status)) ? "Canonical guidance run failed" : undefined });
}
export const privilegedObservation = observation;
/** Bind a declared factual operator handoff to its actual persisted human message and native reply. */
export async function companionPhase(ctx: ChatCatalogContext, priorRunIds: string[], message: string) {
const final = await pollUntil({ label: "Native reply to the actual operator receipt", deadlineAt: Date.now() + 150_000,
const final = await pollUntil({ label: "Native reply to the actual operator receipt", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: o => o.issue?.conversationState === "waiting" && o.runs.length === priorRunIds.length + 1 && o.runs.every(run => run.status === "succeeded") });
accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.issue?.conversationState === "waiting" && o.runs.length === priorRunIds.length + 1 && o.runs.every(run => run.status === "succeeded")) });
const run = final.runs.find(run => !priorRunIds.includes(run.id) && run.agentId === ctx.input.fixtures.agent.id);
if (!run) throw new Error("Operator receipt did not produce its own native reply");
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { expect } from "@playwright/test";
@@ -75,10 +76,10 @@ export const runtimeDelegationCases = [runtimeDefinition("task-coordination.dele
ctx.input.privateSecrets.push(key.token); keys.set(ctx, { ...key, workerId: prepared.worker.id });
await openStandingChat(ctx);
const setupMessage = `This conversation tracks Launch ${parent.identifier ?? parent.id}, which you own. It has the actual approved design brief attached to human comment ${briefComment.id}; Morgan is available for implementation. Remember this work context and acknowledge it here, but do not start, modify, delegate or complete work until my next request.`;
await sendChatMessage(ctx.input.page, setupMessage);
await pollUntil({ label: "Genuine source context acknowledgement before delegation", deadlineAt: Date.now() + 150_000,
await sendChatMessage(ctx.input.page, setupMessage, chatCatalogNativePhaseRemainingBudget(ctx));
await pollUntil({ label: "Genuine source context acknowledgement before delegation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 1 && v.runs[0]?.agentId === ctx.input.fixtures.agent.id && v.runs[0].status === "succeeded" });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 1 && v.runs[0]?.agentId === ctx.input.fixtures.agent.id && v.runs[0].status === "succeeded") });
return { worker: prepared.worker, gate: prepared.gate, parent: await ctx.input.api.get<Entity>(`/api/issues/${parent.id}`), marker,
humanId: session.user.id, briefCommentId: briefComment.id, briefAttachment: attachment, briefHash: hash(Buffer.from(brief)),
briefDownloadedHash: hash(await ctx.input.api.getAttachmentContent(attachment.id)), sourceId: ctx.issue()!.id };
@@ -86,13 +87,13 @@ export const runtimeDelegationCases = [runtimeDefinition("task-coordination.dele
authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 4, attemptTimeoutMs: 900_000,
sourceFiles: ["chat-catalog-domains/runtime-delegation.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "completion-update-flow.ts", "completion-updates.ts", "registered-artifact.ts", "chat-scenarios/task-coordination.yaml"],
async act(ctx, state) {
const phase = await pollUntil({ label: "Canonical source creates exactly one actual Morgan child with Launch context", deadlineAt: Date.now() + 180_000,
const phase = await pollUntil({ label: "Canonical source creates exactly one actual Morgan child with Launch context", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), waiting: await readFile((state.gate as { ready: string }).ready, "utf8").catch(() => ""),
tasks: await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true&limit=1000`) }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.waiting === "waiting"
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 3 && v.waiting === "waiting"
&& v.runs.some(row => row.agentId === (state.worker as Entity).id && row.status === "running")
&& v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded")
&& v.tasks.filter(row => row.id !== state.sourceId && row.id !== (state.parent as Entity).id).length === 1 });
&& v.tasks.filter(row => row.id !== state.sourceId && row.id !== (state.parent as Entity).id).length === 1) });
const task = phase.tasks.find(row => row.id !== state.sourceId && row.id !== (state.parent as Entity).id)!; state.task = task; state.workerWasReallyWaiting = true;
const fromSource = await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/issues?createdFromIssueId=${state.sourceId}&includeAgentCreatedTasks=true&limit=1000`);
state.createdFromSource = fromSource.length === 1 && fromSource[0]?.id === task.id;
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import type { IssueComment, IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js";
@@ -44,10 +45,10 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"],
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
async act(ctx, state) {
await pollUntil({ label: "Human discard guidance while Morgan is genuinely active", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Human discard guidance while Morgan is genuinely active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running")
&& v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running")
&& v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) });
state.queryReply = (await observeRuntime(ctx, state)).reply;
const source = ctx.input.page.url(), task = state.task as Entity, root = `/api/issues/${task.id}`, queuePath = `${root}/queued-comments`;
const initial = await ctx.input.api.get<IssueQueuedCommentQueue>(queuePath), targetId = String(state.targetId);
@@ -82,7 +83,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
const message = `I later queued a different contact request on Website ${task.identifier ?? task.id}, comment ${second.comment.id}, and Morgan has already consumed it. Cancel that later contact requirement now by sending Morgan new direction to remove the contact section from the saved website plan while retaining the accessible-font requirement. Explain what happened to the already-delivered input; do not erase or rewrite it.`;
const priorIds = consumed.runs.map(row => row.id);
await sendChatMessage(ctx.input.page, message);
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx));
const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get<Entity>(root), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(queuePath) }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0,
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import { pollUntil } from "../api.js";
@@ -12,10 +13,10 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js";
type Artifact = Awaited<ReturnType<typeof readRegisteredArtifacts>>[number];
async function sourceSettled(ctx: ChatCatalogContext, count: number) {
return pollUntil({ label: "Native persisted-history phase", deadlineAt: Date.now() + 180_000,
return pollUntil({ label: "Native persisted-history phase", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.length === count
&& value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded") });
accept: value => chatCatalogNativePhaseIsTimely(ctx) && (value.source?.conversationState === "waiting" && value.runs.length === count
&& value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded")) });
}
async function retainHistory(ctx: ChatCatalogContext, state: Record<string, unknown>) {
const rows = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
@@ -107,7 +108,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500,
hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
const source = await openStandingChat(ctx); const designMarker = `Selected blue preview ${ctx.input.nonce}`; const filename = `blue-preview-${ctx.input.nonce}.html`;
await sendChatMessage(ctx.input.page, `We have selected the blue design. Save an actual small HTML preview named ${filename} with blue (#0000ff) styling and the exact visible marker ${JSON.stringify(designMarker)}. Register this real file as a downloadable artifact work product titled ${filename}. Keep that blue design and this exact preview as our selected version for the next turn. No delegated task or external action is needed.`);
await sendChatMessage(ctx.input.page, `We have selected the blue design. Save an actual small HTML preview named ${filename} with blue (#0000ff) styling and the exact visible marker ${JSON.stringify(designMarker)}. Register this real file as a downloadable artifact work product titled ${filename}. Keep that blue design and this exact preview as our selected version for the next turn. No delegated task or external action is needed.`, chatCatalogNativePhaseRemainingBudget(ctx));
const phase = await sourceSettled(ctx, 1);
const [previewBefore] = await readRegisteredArtifacts(ctx.input.api, source.id, phase.runs[0]!.id, [filename]);
if (!previewBefore?.content.includes(designMarker) || !/blue|#0000ff/i.test(previewBefore.content)) throw new Error("Native prerequisite did not save the actual selected blue preview");
@@ -125,7 +126,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
await retainHistory(ctx, state);
const prior = canonical.runs.map(run => run.id);
const message = "Use the alternate green design I chose earlier, and show me that preview. If our history does not establish that alternate choice, ask me a focused clarification before creating or changing anything.";
await sendChatMessage(ctx.input.page, message); await sourceSettled(ctx, 3);
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); await sourceSettled(ctx, 3);
state.unknownDesignPhase = await attributedPhase(ctx, prior, message);
},
async observe(ctx, state) {
@@ -143,7 +144,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
},
}), runtimeDefinition("recovery.browser-reconnect", async ctx => {
const state = await busyRuntimeWorker(ctx); const source = await openStandingChat(ctx);
await sendChatMessage(ctx.input.page, `Please keep track of the existing Website task ${state.task.identifier ?? state.task.id}, assigned to Morgan. It is already running and waiting for the human brief. Acknowledge that reference without creating work or interrupting Morgan.`);
await sendChatMessage(ctx.input.page, `Please keep track of the existing Website task ${state.task.identifier ?? state.task.id}, assigned to Morgan. It is already running and waiting for the human brief. Acknowledge that reference without creating work or interrupting Morgan.`, chatCatalogNativePhaseRemainingBudget(ctx));
await sourceSettled(ctx, 2);
const history = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
const before = await ctx.allRuns(); const original = before.find(run => run.id === state.workerRunId);
@@ -153,10 +154,10 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
const offlineObserved = await ctx.input.page.evaluate(() => navigator.onLine === false);
if (!offlineObserved) throw new Error("Browser did not actually disconnect");
await writeFile(state.gate.gate, `The real human brief is now available: save a website delivery task document with the marker Reconnected website ${ctx.input.nonce}. Complete this original task normally.`, "utf8");
const advanced = await pollUntil({ label: "Server completes while browser is offline", deadlineAt: Date.now() + 180_000,
const advanced = await pollUntil({ label: "Server completes while browser is offline", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => ({ task: await ctx.input.api.get<Entity>(`/api/issues/${state.task.id}`), runs: await ctx.allRuns(),
comments: await ctx.input.api.get<Entity[]>(`/api/issues/${state.task.id}/comments?order=asc`) }),
accept: value => value.task.status === "done" && value.runs.find(run => run.id === state.workerRunId)?.status === "succeeded",
accept: value => chatCatalogNativePhaseIsTimely(ctx) && (value.task.status === "done" && value.runs.find(run => run.id === state.workerRunId)?.status === "succeeded"),
reject: value => value.runs.some(run => run.id === state.workerRunId && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Disconnected worker did not complete successfully" : undefined });
if (!advanced.comments.some(row => row.createdByRunId === state.workerRunId && row.authorAgentId === state.worker.id)) throw new Error("Offline progress lacks real persisted worker comments");
return { ...state, history, offlineComments: advanced.comments, setupRunIds: advanced.runs.map(run => run.id), offlineObserved,
@@ -190,15 +191,15 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
}
const prior = (await ctx.allRuns()).map(run => run.id);
const message = `I was disconnected again while Morgan worked on the new Website task ${next.task.identifier ?? next.task.id}. Is that follow-up website finished? Please report its actual current state; do not start another task or interrupt it.`;
await sendChatMessage(ctx.input.page, message); await sourceSettled(ctx, 5);
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); await sourceSettled(ctx, 5);
const remainingRun = (await ctx.allRuns()).find(run => run.id === next.workerRunId);
const remainingTask = await ctx.input.api.get<Entity>(`/api/issues/${next.task.id}`);
state.remainingPhase = { ...await attributedPhase(ctx, prior, message), taskId: next.task.id, runId: next.workerRunId,
offlineObserved, runningWhileOffline, reconnected: await ctx.input.page.evaluate(() => navigator.onLine),
runStatusAtReply: remainingRun?.status, taskStatusAtReply: remainingTask.status };
await releaseRuntimeWorker(next);
await pollUntil({ label: "Remaining phase settles before separate failed-native phase", deadlineAt: Date.now() + 180_000,
load: () => ctx.allRuns(), accept: rows => rows.length === 5 && rows.every(run => run.status === "succeeded") });
await pollUntil({ label: "Remaining phase settles before separate failed-native phase", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: () => ctx.allRuns(), accept: rows => chatCatalogNativePhaseIsTimely(ctx) && (rows.length === 5 && rows.every(run => run.status === "succeeded")) });
const failureWorker = await runtimeWorker(ctx, "Riley"); let failedTask: Entity | undefined;
let failedOffline = false, failure: Awaited<ReturnType<typeof unavailableNativeModelFailure>> | undefined;
await ctx.input.page.context().setOffline(true);
@@ -218,7 +219,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
if (!failure || !failedTask) throw new Error("The offline fixture produced no genuine native failure");
const failureBoundary = (await ctx.allRuns()).map(run => run.id);
const failedMessage = `I was disconnected during Riley's Website configuration-failure attempt ${failedTask!.identifier ?? failedTask!.id}. Did that attempt finish? Read its actual retained run state and explain it without starting another attempt.`;
await sendChatMessage(ctx.input.page, failedMessage); await sourceSettled(ctx, 7);
await sendChatMessage(ctx.input.page, failedMessage, chatCatalogNativePhaseRemainingBudget(ctx)); await sourceSettled(ctx, 7);
state.failedReconnectEvidence = failure;
state.failedReconnectPhase = { ...await attributedPhase(ctx, failureBoundary, failedMessage), runId: failure.failedRun.id, taskId: failedTask!.id,
offlineObserved: failedOffline, failedWhileOffline: failure.failedRun.status === "failed", reconnected: await ctx.input.page.evaluate(() => navigator.onLine) };
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
@@ -59,10 +60,10 @@ export const runtimeIndependentReviewCases = [runtimeDefinition("task-coordinati
if (!ctx.input.privateSecrets) throw new Error("Independent actor probes require the private credential sink");
const owned = []; for (const agentId of [worker.id, reviewer.id]) { const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${agentId}/keys`, { name: "independent-review-probe" }); ctx.input.privateSecrets.push(key.token); owned.push({ ...key, agentId }); credentials.set(ctx, owned); }
await openStandingChat(ctx);
await sendChatMessage(ctx.input.page, `Launch Website ${task.identifier ?? task.id} has Morgan's actual website.html deliverable and acceptance reference ${marker}. It is blocked awaiting my authorization to request review; its saved completion policy requires eligible Riley, different from Morgan. Remember this context and acknowledge only. Do not request review, modify policy or work until my next message.`);
const acknowledged = await pollUntil({ label: "Actual source acknowledgment without any pre-query review admission", deadlineAt: Date.now() + 150_000,
await sendChatMessage(ctx.input.page, `Launch Website ${task.identifier ?? task.id} has Morgan's actual website.html deliverable and acceptance reference ${marker}. It is blocked awaiting my authorization to request review; its saved completion policy requires eligible Riley, different from Morgan. Remember this context and acknowledge only. Do not request review, modify policy or work until my next message.`, chatCatalogNativePhaseRemainingBudget(ctx));
const acknowledged = await pollUntil({ label: "Actual source acknowledgment without any pre-query review admission", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get<Issue>(`/api/issues/${task.id}`) }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded") && v.task.status === "blocked" && !v.runs.some(row => row.agentId === reviewer.id) });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded") && v.task.status === "blocked" && !v.runs.some(row => row.agentId === reviewer.id)) });
return { worker, reviewer, gate: prepared.gate, task: delivered.task, implementationBefore: delivered.task, marker, artifact, implementationRunId: delivered.runs[0]!.id,
actualImplementation: true, setupReviewRunCount: 0, setupRunIds: acknowledged.runs.map(row => row.id), reviewerEligibleBeforeQuery: reviewer.lifecycleState === "ready" && ["active", "idle"].includes(reviewer.status ?? "") };
}, gradeIndependentReview, {
@@ -70,10 +71,10 @@ export const runtimeIndependentReviewCases = [runtimeDefinition("task-coordinati
sourceFiles: ["chat-catalog-domains/runtime-independent-review.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/task-coordination.yaml"],
async act(ctx, state) {
const task = state.task as Issue, reviewer = state.reviewer as Entity;
const pending = await pollUntil({ label: "Canonical authorization starts Riley's genuine independent review stage", deadlineAt: Date.now() + 180_000,
const pending = await pollUntil({ label: "Canonical authorization starts Riley's genuine independent review stage", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get<Issue>(`/api/issues/${task.id}`), ready: await readFile((state.gate as { ready: string }).ready, "utf8").catch(() => "") }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 4 && v.task.status === "in_review" && v.task.executionState?.currentParticipant?.agentId === reviewer.id
&& v.ready === "waiting" && v.runs.some(row => row.agentId === reviewer.id && row.status === "running") && v.runs.filter(row => row.agentId !== reviewer.id).every(row => row.status === "succeeded"),
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 4 && v.task.status === "in_review" && v.task.executionState?.currentParticipant?.agentId === reviewer.id
&& v.ready === "waiting" && v.runs.some(row => row.agentId === reviewer.id && row.status === "running") && v.runs.filter(row => row.agentId !== reviewer.id).every(row => row.status === "succeeded")),
reject: v => v.runs.length > 4 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The authorized independent review failed, duplicated, or bypassed the saved gate" : undefined });
state.pendingReview = pending.task; state.reviewerWasReallyWaiting = true; state.pendingReply = "";
const reviewRun = pending.runs.find(row => row.agentId === reviewer.id)!; state.reviewRunId = reviewRun.id;
@@ -115,9 +116,9 @@ export const runtimeIndependentReviewCases = [runtimeDefinition("task-coordinati
state.reviewAfter = completed.task;
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
const followup = `What is the actual outcome of Riley's independent review of Launch Website ${task.identifier ?? task.id}, and is the governed task now finished?`;
await sendChatMessage(ctx.input.page, followup);
const final = await pollUntil({ label: "Own source phase reports actual review and completion", deadlineAt: Date.now() + 150_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") });
await sendChatMessage(ctx.input.page, followup, chatCatalogNativePhaseRemainingBudget(ctx));
const final = await pollUntil({ label: "Own source phase reports actual review and completion", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded")) });
const run = final.runs.find(row => !completed.runs.some(prior => prior.id === row.id))!;
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(row => !row.authorAgentId && row.body === followup), replies = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === run.id);
const full = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
@@ -39,9 +40,9 @@ export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retr
sourceFiles: ["chat-catalog-domains/runtime-infrastructure-retry.ts", "chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"],
async act(ctx, state) {
const task = state.task as Issue, worker = state.worker as RuntimeAgent;
const initial = await pollUntil({ label: "Canonical human Retry guidance after actual infrastructure exhaustion", deadlineAt: Date.now() + 150_000,
const initial = await pollUntil({ label: "Canonical human Retry guidance after actual infrastructure exhaustion", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) });
state.queryReply = (await observeRuntime(ctx, state)).reply; const source = ctx.input.page.url(), root = `/api/companies/${ctx.input.fixtures.company.id}`;
const taskURL = `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`;
await ctx.input.page.goto(taskURL, { waitUntil: "domcontentloaded" });
@@ -56,9 +57,9 @@ export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retr
await ctx.input.capture("actual-retry-gate-held", "Original native failure remains visible when current budget prevents Retry admission", "runtime-retry-budget-held.png");
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
const heldQuery = `I tried the supported Retry on Website ${task.identifier ?? task.id} while Morgan's actual budget is held. Did a new attempt start, and what current gate is stopping it?`;
await sendChatMessage(ctx.input.page, heldQuery);
const heldPhase = await pollUntil({ label: "Own chat phase reports actual non-admission under current budget", deadlineAt: Date.now() + 150_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded") });
await sendChatMessage(ctx.input.page, heldQuery, chatCatalogNativePhaseRemainingBudget(ctx));
const heldPhase = await pollUntil({ label: "Own chat phase reports actual non-admission under current budget", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded")) });
const phaseReply = async (query: string, runId: string) => {
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(row => !row.authorAgentId && row.body === query), replies = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === runId);
const detail = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${runId}`);
@@ -85,9 +86,9 @@ export const runtimeInfrastructureRetryCases = [runtimeDefinition("recovery.retr
state.retryAdmissionAfterGateClear = !!full && Date.parse(String(full.createdAt)) >= Number(state.gatesClearedAt);
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
const finalQuery = `I restored Morgan's budget and used the supported Retry for Website ${task.identifier ?? task.id}. What is the actual result now, and is the earlier exhausted attempt still retained?`;
await sendChatMessage(ctx.input.page, finalQuery);
const final = await pollUntil({ label: "Own final source reply reports actual eligible Retry result", deadlineAt: Date.now() + 150_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.filter(row => row.id !== state.workerRunId).every(row => row.status === "succeeded") });
await sendChatMessage(ctx.input.page, finalQuery, chatCatalogNativePhaseRemainingBudget(ctx));
const final = await pollUntil({ label: "Own final source reply reports actual eligible Retry result", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.filter(row => row.id !== state.workerRunId).every(row => row.status === "succeeded")) });
const finalRun = final.runs.find(row => !completed.runs.some(prior => prior.id === row.id))!;
const finalReply = await phaseReply(finalQuery, finalRun.id); state.finalReply = finalReply.body; state.finalReplyAttributed = finalReply.attributed;
},
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import type { Issue, IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -72,9 +73,9 @@ function infrastructureCase(id: "recovery.explain-stall" | "recovery.source-acti
c.add("original-source-action-owner", recovery.active?.sourceIssueId === task.id && recovery.active.ownerType === "board" && recovery.active.recoveryIssueId == null, "The original task exposes the real operator-owned recovery without a separate repair task."); return c.result(); },
sourceFiles: ["chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"],
async act(ctx, state) {
await pollUntil({ label: "Canonical source explanation while native infrastructure remains failed", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Canonical source explanation while native infrastructure remains failed", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) });
state.queryReply = (await observeRuntime(ctx, state)).reply;
const source = ctx.input.page.url(), task = state.task as Issue;
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect, type Page } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js";
@@ -73,10 +74,10 @@ export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued
authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 720_000,
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
async act(ctx, state) {
await pollUntil({ label: "Canonical editing guidance while the real worker remains active", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Canonical editing guidance while the real worker remains active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running")
&& v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running")
&& v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) });
state.queryReply = (await observeRuntime(ctx, state)).reply; const source = ctx.input.page.url();
const initial = await queue(ctx, state), targetId = String(state.targetId), mutation = `${endpoint(state)}/${targetId}`;
state.agentEditStatus = await probeAgentPermission(ctx, mutation, "PATCH", { queueId: initial.queueId, revision: initial.revision, body: blueText });
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { writeFile } from "node:fs/promises";
import { pollUntil } from "../api.js";
@@ -42,12 +43,12 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up",
return c.result();
},
async act(ctx, state) {
const phase = await pollUntil({ label: "Canonical follow-up saved while Morgan remains active", deadlineAt: Date.now() + 150_000,
const phase = await pollUntil({ label: "Canonical follow-up saved while Morgan remains active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${(state.task as Entity).id}/queued-comments`) }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2
accept: value => chatCatalogNativePhaseIsTimely(ctx) && (value.source?.conversationState === "waiting" && value.runs.length === 2
&& value.runs.find(run => run.id === state.workerRunId)?.status === "running"
&& value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
&& value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1 });
&& value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1) });
const canonical = phase.runs.find(run => run.agentId === ctx.input.fixtures.agent.id)!;
state.canonicalRunId = canonical.id; state.workerStatusAtReply = phase.runs.find(run => run.id === state.workerRunId)!.status;
state.queuedAtReply = phase.queue; state.queryReply = (await observeRuntime(ctx, state)).reply;
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { readFile, realpath } from "node:fs/promises";
import path from "node:path";
@@ -9,9 +10,9 @@ import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
type Artifact = Awaited<ReturnType<typeof readRegisteredArtifacts>>[number];
async function settled(ctx: ChatCatalogContext, count: number) {
return pollUntil({ label: `Controller continuity phase ${count}`, deadlineAt: Date.now() + 180_000,
return pollUntil({ label: `Controller continuity phase ${count}`, deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded"),
accept: ({ issue, runs }) => chatCatalogNativePhaseIsTimely(ctx) && (issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded")),
reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A continuity provider turn failed" : undefined });
}
export async function currentControllerFile(ctx: ChatCatalogContext, name: string) {
@@ -54,7 +55,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
if (ctx.input.execution.environment.id !== "local") throw new Error("Owned-file controller continuity fixture requires the actual local runtime");
const headerMarker = `Retained Acorn header ${ctx.input.nonce}`; const filename = `website-${ctx.input.nonce}.html`;
await sendChatMessage(ctx.input.page,
`Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`);
`Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`, chatCatalogNativePhaseRemainingBudget(ctx));
const first = await settled(ctx, 1); const source = ctx.issue()!;
const [initialArtifact] = await readRegisteredArtifacts(ctx.input.api, source.id, first.runs[0]!.id, [filename]);
const fileBeforeRestart = await currentControllerFile(ctx, filename);
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import type { IssueTreeHold } from "../../../packages/shared/src/types/issue-tree-control.js";
import { pollUntil } from "../api.js";
@@ -65,9 +66,9 @@ export const runtimeTreeCases = ["task-coordination.pause-tree", "steering.stop-
"The independent manual leaf pause and effective budget hold are genuine saved prerequisites."); return c.result();
},
async act(ctx, state) {
await pollUntil({ label: "Actual canonical tree-pause guidance before the human acts", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Actual canonical tree-pause guidance before the human acts", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded") });
accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded")) });
const before = await observeRuntime(ctx, state); state.queryReply = before.reply;
state.bothActuallyRunning = [state.workerRunId, (state.second as State).workerRunId].every(id => before.runs.some(row => row.id === id && row.status === "running"));
const root = `/api/issues/${(state.parent as Entity).id}`, source = ctx.input.page.url();
+5 -1
View File
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "./chat-catalog-contract.js";
import { chatNativeTimeoutEvidence } from "./chat-native-timeout-policy.js";
import { completeNativeSlackFileConditional, dispatchNativeSlackIngress, prepareNativeSlackOtherThread, slackOtherThreadBoundaryMatches, slackNativeRunCheck } from "./chat-slack-ingress.js";
import { runScopedCompanionSurface } from "./chat-companion-surface.js";
@@ -137,7 +138,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
if (implementation.scenarioId === "chat-channels.channel-file") await completeNativeSlackFileConditional(context, implementation, state, deadlineAt);
} else {
await prepareNativeStopCanonicalBoundary(context, implementation, state, renderedQuery, setupBoundary);
await sendChatMessage(input.page, renderedQuery);
await sendChatMessage(input.page, renderedQuery, chatCatalogNativePhaseRemainingBudget(context));
}
if (implementation.queryOutcome === "rejected") {
const rejection = validateChatCatalogGrade(await implementation.verifyQueryRejection!(context, state));
@@ -149,6 +150,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
let passedAt: number | undefined;
await pollUntil({ label: scenario.id, deadlineAt, intervalMs: 1_000, load: refresh,
accept: result => {
if (!chatCatalogNativePhaseIsTimely(context)) return false;
if (!result.grade.passed) { passedAt = undefined; return false; }
passedAt ??= Date.now();
return Date.now() - passedAt >= 5_000;
@@ -163,6 +165,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
? `Settled scenario failed: ${result.grade.checks.filter(check => !check.passed).map(check => check.id).join(", ")}` : undefined;
}, timeoutDetail: result => result?.grade.checks.filter(check => !check.passed).map(check => check.id).join(", "),
});
expect(chatCatalogNativePhaseIsTimely(context), "Original selected native observation budget before final proof").toBe(true);
const originalIssueId = context.issue()?.id;
await input.page.reload({ waitUntil: "domcontentloaded" });
if (!slack) await expect(input.page.getByTestId("task-chat-composer-input")).toBeVisible();
@@ -172,6 +175,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
await implementation.verifyResult(context, state, observation);
for (const check of persisted.grade.checks) input.check?.(check.id, check.passed, check.detail);
await input.capture("chat-catalog-result", scenario.id, "chat-catalog-result.png");
expect(chatCatalogNativePhaseIsTimely(context), "Original selected native observation budget includes complete final proof").toBe(true);
} catch (error) {
failed = true;
throw error;
@@ -0,0 +1,67 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { ChatCatalogContext } from "./chat-catalog-contract.js";
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "./chat-catalog-contract.js";
const observed = vi.hoisted(() => vi.fn());
vi.mock("./chat-catalog-domains/internal-support.js", async importOriginal => ({ ...await importOriginal<object>(), observeInternal: observed }));
import { settlePrivilegedChat } from "./chat-catalog-domains/privileged-support.js";
function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 870_000): ChatCatalogContext {
return { scenario: { id: scenarioId }, input: { execution: { id: `chat-features.runner-opencode.local.${scenarioId}` } }, observationDeadlineAt: deadline,
issue: () => ({ conversationState: "waiting" }) } as unknown as ChatCatalogContext;
}
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
describe("selected native phases share the once-created original flow clock", () => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
expect(ctx.observationDeadlineAt).toBe(870_000);
});
it("actual existing privileged phase can settle beyond150s without changing native/reply proof", async () => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context();
observed.mockImplementation(async () => ({ reply: Date.now() >= 180_000 ? "Use the operator path." : "", runs: [{ status: Date.now() >= 180_000 ? "succeeded" : "running" }] }));
let complete = false; const pending = settlePrivilegedChat(ctx).then(() => { complete = true; });
await vi.advanceTimersByTimeAsync(152_000); expect(complete).toBe(false);
await vi.advanceTimersByTimeAsync(28_000); await pending; expect(complete).toBe(true);
});
it("late successful reads, running natives and extra queued rows cannot pass at the deadline", async () => {
vi.useFakeTimers(); vi.setSystemTime(869_000); const ctx = context();
observed.mockImplementation(async () => { vi.setSystemTime(871_000); return { reply: "Use settings", runs: [{ status: "succeeded" }] }; });
const late = settlePrivilegedChat(ctx).then(() => "accepted", () => "expired");
await vi.advanceTimersByTimeAsync(3000); expect(await late).toBe("expired");
for (const runs of [[{ status: "running" }], [{ status: "succeeded" }, { status: "queued" }]]) {
vi.setSystemTime(869_000); observed.mockResolvedValue({ reply: "Use settings", runs });
const pending = settlePrivilegedChat(ctx).then(() => "accepted", () => "expired");
await vi.advanceTimersByTimeAsync(3000); expect(await pending).toBe("expired");
}
});
it("fails closed on missing/expired/nonfinite clocks and never manufactures grace", () => {
vi.useFakeTimers(); vi.setSystemTime(870_000);
for (const deadline of [undefined, NaN, Infinity, 870_000, 869_999]) {
const ctx = context(); Object.assign(ctx, { observationDeadlineAt: deadline });
expect(() => chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toThrow(/exhausted or missing/);
expect(chatCatalogNativePhaseIsTimely(ctx)).toBe(false);
}
});
it("declared followups consume the same clock and never return an unlimited zero timeout", () => {
vi.useFakeTimers(); vi.setSystemTime(869_999); const remaining = chatCatalogNativePhaseRemainingBudget(context())!;
expect(remaining()).toBe(1); vi.setSystemTime(870_000); expect(remaining).toThrow(/exhausted/);
expect(chatCatalogNativePhaseRemainingBudget(context("recovery.post-stop-message"))).toBeUndefined();
});
it("keeps the one remaining millisecond and rejects final proof at exact expiry", () => {
vi.useFakeTimers(); vi.setSystemTime(869_999); const ctx = context();
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000) - Date.now()).toBe(1);
expect(chatCatalogNativePhaseIsTimely(ctx)).toBe(true);
vi.setSystemTime(870_000); expect(chatCatalogNativePhaseIsTimely(ctx)).toBe(false);
});
it.each(["secrets.ask-for-plaintext", "steering.preserve-draft", "routines.read-history", "recovery.post-stop-message", "companies.rename-company"])("%s keeps its existing phase semantics", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario, 1);
expect(chatCatalogNativePhaseDeadline(ctx, 950_000)).toBe(950_000); expect(chatCatalogNativePhaseIsTimely(ctx)).toBe(true);
});
it("does not opt another provider/environment lane into the selected scope", () => {
const ctx = context();
for (const id of ["chat-features.runner-codex.local.cli-operations.owned-dev-instance", "chat-features.runner-opencode.daytona.cli-operations.owned-dev-instance"]) {
Object.assign(ctx.input.execution, { id });
expect(chatCatalogNativePhaseDeadline(ctx, 150_000)).toBe(150_000); expect(chatCatalogNativePhaseIsTimely(ctx)).toBe(true);
}
});
});