From ccb6133d641f6f337639c5188d8ffd7e8769838e Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 08:07:17 -0500 Subject: [PATCH] fix(evals): bind post-stop readiness to the actual source workspace --- .../runtime-post-stop-fixture.test.ts | 37 +++++++++++++++++++ .../chat-catalog-domains/runtime-post-stop.ts | 12 +++--- 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-post-stop-fixture.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop-fixture.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop-fixture.test.ts new file mode 100644 index 00000000000..199f1548d86 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop-fixture.test.ts @@ -0,0 +1,37 @@ +import { expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { openStandingChat } from "./identity-support.js"; +import { prepareRuntimeWorkerGate, readRuntimeTaskGate, taskBoundRuntimeGate } from "./runtime-busy-worker.js"; +import { runtimePostStopCases } from "./runtime-post-stop.js"; + +vi.mock("../chat-flow.js", async original => ({ ...await original(), sendChatMessage: vi.fn() })); +vi.mock("./identity-support.js", async original => ({ ...await original(), openStandingChat: vi.fn() })); +vi.mock("./runtime-busy-worker.js", async original => ({ ...await original(), + prepareRuntimeWorkerGate: vi.fn(), readRuntimeTaskGate: vi.fn() })); + +it("waits for the task-bound listener on the genuine source and lead before preparing Stop", async () => { + const lead = { id: "source-agent", companyId: "company", lifecycleState: "ready" }; + const source = { id: "canonical-source" }; + const gate = taskBoundRuntimeGate("same-source-post-stop"); + const close = vi.fn(async () => {}); + const secondary = { close }; + vi.mocked(openStandingChat).mockResolvedValue(source as Awaited>); + vi.mocked(prepareRuntimeWorkerGate).mockResolvedValue({ worker: lead, gate, previousRuns: [], company: "/api/companies/company" }); + vi.mocked(readRuntimeTaskGate).mockResolvedValue("waiting"); + vi.mocked(sendChatMessage).mockResolvedValue(undefined); + const ctx = { input: { fixtures: { company: { id: "company" }, agent: lead }, page: { context: () => ({ newPage: async () => secondary }) }, + api: { get: async (route: string) => { + if (route === "/api/agents/source-agent") return lead; + if (route === "/api/heartbeat-runs/original-native") throw new Error("independent run identity read reached"); + throw new Error("Unexpected fixture read"); + } } }, allRuns: async () => [{ id: "original-native", agentId: lead.id, status: "running" }] } as unknown as ChatCatalogContext; + await expect(runtimePostStopCases[0]!.setup(ctx)).rejects.toThrow("independent run identity read reached"); + expect(prepareRuntimeWorkerGate).toHaveBeenCalledWith(ctx, { worker: lead, taskBound: true }); + expect(readRuntimeTaskGate).toHaveBeenCalledWith(ctx, gate, source.id, lead.id); + expect(sendChatMessage).toHaveBeenCalledOnce(); + const firstHumanRequest = vi.mocked(sendChatMessage).mock.calls[0]![1]; + expect(firstHumanRequest).toContain(`First run the bounded command ${gate.command} once`); + expect(firstHumanRequest).not.toContain("wait-for-brief-runtime-"); + expect(close).toHaveBeenCalledOnce(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts index 4cf9d0c2614..3a79e895f65 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts @@ -1,5 +1,5 @@ import { expect, type Page } from "@playwright/test"; -import { readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import type { ChatCatalogContext, NativeStopBoundaryReceipt } from "../chat-catalog-contract.js"; import type { IssueComment } from "../../../packages/shared/src/types/issue.js"; import { pollUntil } from "../api.js"; @@ -8,7 +8,7 @@ import { collectRunEvents } from "../run-observations.js"; import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks } from "./internal-support.js"; import { openStandingChat } from "./identity-support.js"; -import { prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; +import { prepareRuntimeWorkerGate, readRuntimeTaskGate } from "./runtime-busy-worker.js"; import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited } from "./runtime-provider-cleanup.js"; import type { NativeFaultRun } from "./runtime-native-fault.js"; import { hasSimplerWebsite, renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; @@ -48,13 +48,13 @@ export function gradePostStop(value: unknown) { } export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-message", async ctx => { const source = await openStandingChat(ctx), lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); - const { gate } = await prepareRuntimeWorkerGate(ctx, { worker: lead }); + const { gate } = await prepareRuntimeWorkerGate(ctx, { worker: lead, taskBound: true }); const secondary = await ctx.input.page.context().newPage(); handles.set(ctx, { page: secondary }); try { - const first = `Build an Acorn Notes website with a visible heading, product paragraph and labelled signup action. Initially use several panels and tabs. Work directly in this conversation; save and register the actual website.html artifact here, without delegating another task. First run the bounded command node ${gate.scriptPath} once and wait for its output before implementing the website. Do not replace the command or invent its output. Preserve later human changes to this same website.`; + const first = `Build an Acorn Notes website with a visible heading, product paragraph and labelled signup action. Initially use several panels and tabs. Work directly in this conversation; save and register the actual website.html artifact here, without delegating another task. First run the bounded command ${gate.command} once and wait for its output before implementing the website. Do not replace the command or invent its output. Preserve later human changes to this same website.`; await sendChatMessage(ctx.input.page, first); const running = await pollUntil({ label: "Genuine initial source Website request executing its bounded native tool", deadlineAt: Date.now() + 150000, - load: async () => ({ runs: await ctx.allRuns(), waiting: await readFile(gate.ready, "utf8").catch(() => "") }), + load: async () => ({ runs: await ctx.allRuns(), waiting: await readRuntimeTaskGate(ctx, gate, source.id, lead.id) }), accept: v => v.waiting === "waiting" && v.runs.length === 1 && v.runs[0]?.agentId === lead.id && v.runs[0].status === "running", reject: v => v.runs.some(row => row.status !== "running" && row.status !== "queued") ? "The genuine initial source execution ended before Stop setup" : undefined }); const original = await ctx.input.api.get(`/api/heartbeat-runs/${running.runs[0]!.id}`); @@ -64,7 +64,7 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa if (initial.length !== 1 || initial[0]?.body !== first || initial[0].authorUserId !== people.user.id) throw new Error("The initial Website request is not the exact genuine signed-in human setup turn"); await secondary.goto(ctx.input.page.url(), { waitUntil: "domcontentloaded" }); await expect(secondary.getByTestId("task-chat-composer-stop")).toBeEnabled(); return { gate, sourceId: source.id, workerRunId: original.id, initialHuman: initial[0], humanId: people.user.id }; - } catch (error) { await writeFile(gate.gate, "The pending fixture tool has been released.", "utf8"); await secondary.close(); handles.delete(ctx); throw error; } + } catch (error) { if (gate.gate) await writeFile(gate.gate, "The pending fixture tool has been released.", "utf8"); await secondary.close(); handles.delete(ctx); throw error; } }, gradePostStop, { canonicalBoundary: "native-stop-cleanup", authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 720000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 1 } },