test(evals): cover real preview lifecycle and queued requirement delivery

tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json

@@ -206,8 +206,8 @@
     },
     {
       "scenarioId": "steering.queue-follow-up",
-      "readinessClass": "local",
-      "needs": "Real busy Morgan; save once correct task queue; order/wait state and actual later delivery."
+      "readinessClass": "implemented",
+      "needs": "Actual busy native Morgan; canonical contact instruction saved once into correct ordered task queue with source-run attribution, visible waiting UI; one normal later native input consumes exact saved comment and preserves initial document revision plus contact result."
     },
     {
       "scenarioId": "steering.edit-queued-message",
@@ -286,8 +286,8 @@
     },
     {
       "scenarioId": "workspaces.preview-service",
-      "readinessClass": "local",
-      "needs": "Owned ready workspace with real website files; start supported service; independent readiness HTTP/endpoint view plus startup-failure branch."
+      "readinessClass": "implemented",
+      "needs": "Actual native HTML prerequisite; preconfigured stopped managed service, canonical start, independent HTTP/content/browser/service/work-product identity, explicit actual failed restart and phase-attributed actionable error."
     },
     {
       "scenarioId": "workspaces.inspect-git",

tests/runner-e2e/chat-catalog-domains/runtime-preview.test.ts

@@ -0,0 +1,20 @@
+import { it, expect } from "vitest";
+import { gradePreview } from "./runtime-preview.js";
+function valid() { const url = "http://127.0.0.1:43210/";
+  return { scenarioId: "workspaces.preview-service", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
+    state: { workspace: { id: "actual" }, task: { id: "website" }, marker: "Acorn marker", websiteBytes: "<html><h1>Acorn marker</h1></html>",
+      initialHttpBody: "<html><h1>Acorn marker</h1></html>", initialHttpStatus: 200, browserPreviewVisible: true,
+      healthyService: { id: "service", executionWorkspaceId: "actual", companyId: "company", status: "running", healthStatus: "healthy", serviceName: "Website preview", url },
+      queryLinks: [url], previewProducts: [{ id: "preview", executionWorkspaceId: "actual", runtimeServiceId: "service", url, type: "preview_url", createdByRunId: "canonical" }],
+      failureOperation: { id: "startup-failure", status: "failed", executionWorkspaceId: "actual", heartbeatRunId: "restart" },
+      failureRunIds: ["restart"], failedHttpReachable: false, failureReply: "The preview restart failed because its server process exited. Inspect the startup error and repair it before starting again." },
+    agents: [], tasks: [{ id: "website" }], links: [], activity: [], humanQueries: [], reply: "",
+    runs: [{ id: "website-setup", agentId: "morgan", status: "succeeded" }, { id: "canonical", agentId: "ceo", status: "succeeded" }, { id: "restart", agentId: "ceo", status: "succeeded" }] }; }
+it("accepts actual ready HTTP preview then a separately observed failed restart", () => expect(gradePreview(valid()).passed).toBe(true));
+it("rejects missing real preview evidence", () => expect(gradePreview(undefined).passed).toBe(false));
+it("rejects a service record without HTTP content readiness", () => { const o = valid(); o.state.initialHttpStatus = 503; expect(gradePreview(o).passed).toBe(false); });
+it("rejects serving another website's bytes", () => { const o = valid(); o.state.initialHttpBody = "Other website"; expect(gradePreview(o).passed).toBe(false); });
+it("rejects a similarly named service from another workspace", () => { const o = valid(); o.state.healthyService.executionWorkspaceId = "other"; expect(gradePreview(o).passed).toBe(false); });
+it("rejects a preview product from an unrelated run", () => { const o = valid(); o.state.previewProducts[0]!.createdByRunId = "foreign"; expect(gradePreview(o).passed).toBe(false); });
+it("rejects claiming readiness after the actual startup failure", () => { const o = valid(); o.state.failureReply = "The preview is now ready."; expect(gradePreview(o).passed).toBe(false); });
+it("rejects failure attributed to the initial healthy turn", () => { const o = valid(); o.state.failureOperation.heartbeatRunId = "canonical"; expect(gradePreview(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-preview.ts

@@ -0,0 +1,106 @@
+import { expect } from "@playwright/test";
+import { readFile, realpath, writeFile, unlink } from "node:fs/promises";
+import path from "node:path";
+import { pollUntil } from "../api.js";
+import { sendChatMessage } from "../chat-flow.js";
+import { checks, type Entity } from "./internal-support.js";
+import { savedTaskWorkspace } from "./runtime-workspaces.js";
+import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
+import type { WorkspaceRuntimeService } from "../../../packages/shared/src/types/workspace-runtime.js";
+import type { WorkspaceOperation } from "../../../packages/shared/src/types/workspace-operation.js";
+import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-product.js";
+interface PreviewWorkspace extends Entity { cwd: string; config: Record<string, unknown>; runtimeServices?: WorkspaceRuntimeService[] }
+export function gradePreview(value: unknown) {
+  const c = checks(); const o = runtimeObservation(value);
+  c.add("actual-preview-lifecycle-evidence", !!o, "Actual service identity, HTTP content, source output and subsequent startup failure must be independently observed."); if (!o) return c.result();
+  const service = o.state.healthyService as WorkspaceRuntimeService | undefined; const workspace = o.state.workspace as PreviewWorkspace;
+  const products = o.state.previewProducts as IssueWorkProduct[] | undefined;
+  c.add("real-ready-service-correct-workspace", !!service && service.executionWorkspaceId === workspace.id && service.companyId === o.companyId
+    && service.status === "running" && service.healthStatus === "healthy" && service.serviceName === "Website preview"
+    && o.state.initialHttpStatus === 200 && o.state.initialHttpBody === o.state.websiteBytes
+    && typeof o.state.initialHttpBody === "string" && o.state.initialHttpBody.includes(String(o.state.marker)) && o.state.browserPreviewVisible === true,
+  "A real managed Website preview serves the actual provider-written website bytes in the intended workspace and is usable in the browser.");
+  c.add("accessible-registered-preview", !!service?.url && Array.isArray(products) && products.some(row => row.executionWorkspaceId === workspace.id
+    && row.runtimeServiceId === service.id && row.url === service.url && ["preview_url", "runtime_service"].includes(row.type)
+    && o.runs.some(run => run.id === row.createdByRunId && run.agentId === o.leadId))
+    && (o.state.queryLinks as string[] | undefined)?.includes(service.url) === true,
+  "The actual canonical reply exposes the ready endpoint and its run-attributed work product is tied to this exact workspace/service.");
+  const failure = o.state.failureOperation as WorkspaceOperation | undefined; const reply = String(o.state.failureReply ?? "");
+  c.add("startup-failure-exposed-actionably", !!failure && failure.status === "failed" && failure.executionWorkspaceId === workspace.id
+    && (o.state.failureRunIds as string[] | undefined)?.includes(failure.heartbeatRunId ?? "") === true
+    && o.state.failedHttpReachable === false && /fail|error|could not|couldn't|did not start|not running/i.test(reply)
+    && /start|process|server|preview|exit/i.test(reply) && !/preview (?:is|has) (?:now )?(?:ready|running)|successfully restart/i.test(reply),
+  "A separate explicit restart really fails under the fixture's process-start fault; its own answer preserves the failure and repair context.");
+  c.add("no-unrelated-work-or-publication", o.tasks.length === 1 && o.tasks[0]?.id === (o.state.task as Entity).id
+    && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded"),
+  "The prepared website plus two explicit controller turns are the only native work; no remote deployment or extra task is substituted.");
+  return c.result();
+}
+export const runtimePreviewCases = [runtimeDefinition("workspaces.preview-service", async ctx => {
+  if (ctx.input.execution.environment.id !== "local") throw new Error("Owned loopback website preview fixture requires the actual local target");
+  const state = await savedTaskWorkspace(ctx, { html: true }); const cwd = await realpath(state.workspaceBefore.workspace.cwd);
+  const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
+  if (!root || !cwd.startsWith(`${await realpath(root)}${path.sep}`)) throw new Error("Preview fixture escaped the isolated task directory");
+  const websiteBytes = await readFile(path.join(cwd, "index.html"), "utf8");
+  if (!/<html|<!doctype html/i.test(websiteBytes) || !websiteBytes.includes(state.marker)) throw new Error("Actual native website prerequisite is not present");
+  const failureFlag = path.join(cwd, "preview-start-failure.fixture"); const script = path.join(cwd, "preview-server.cjs");
+  await writeFile(script, `const fs=require('node:fs');const http=require('node:http');const path=require('node:path');if(fs.existsSync(path.join(__dirname,'preview-start-failure.fixture'))){console.error('Owned preview startup fixture failed with exit 17');process.exit(17)}const body=fs.readFileSync(path.join(__dirname,'index.html'));http.createServer((req,res)=>{res.writeHead(200,{'content-type':'text/html; charset=utf-8'});res.end(body)}).listen(Number(process.argv[2]),'127.0.0.1');`, "utf8");
+  const workspace = await ctx.input.api.get<PreviewWorkspace>(`/api/execution-workspaces/${state.workspaceBefore.workspace.id}`);
+  await ctx.input.api.patch(`/api/execution-workspaces/${workspace.id}`, { config: { ...workspace.config, desiredState: "stopped", workspaceRuntime: { services: [{
+    name: "Website preview", command: "node preview-server.cjs {{port}}", cwd: ".", port: { type: "auto" }, lifecycle: "shared", stopPolicy: { type: "manual" },
+    readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 100 },
+    expose: { type: "url", urlTemplate: "http://127.0.0.1:{{port}}" },
+  }] } } });
+  const saved = await ctx.input.api.get<PreviewWorkspace>(`/api/execution-workspaces/${workspace.id}`);
+  if (saved.runtimeServices?.some(row => row.status === "running")) throw new Error("Preview setup must not start the target service");
+  return { ...state, workspace: saved, websiteBytes, failureFlag };
+}, gradePreview, {
+  sourceFiles: ["chat-catalog-domains/runtime-preview.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml"],
+  expectedHumanMessageCount: 2, expectedRunCount: 3, attemptTimeoutMs: 720_000,
+  async act(ctx, state) {
+    await pollUntil({ label: "Actual canonical preview service readiness", deadlineAt: Date.now() + 180_000,
+      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), workspace: await ctx.input.api.get<PreviewWorkspace>(`/api/execution-workspaces/${(state.workspace as Entity).id}`) }; },
+      accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded")
+        && value.workspace.runtimeServices?.some(service => service.status === "running" && service.healthStatus === "healthy") === true });
+    const ws = await ctx.input.api.get<PreviewWorkspace>(`/api/execution-workspaces/${(state.workspace as Entity).id}`);
+    const service = ws.runtimeServices?.find(row => row.serviceName === "Website preview" && row.status === "running")!;
+    if (!service?.url) throw new Error("A created service record without an accessible ready endpoint is not a preview");
+    const url = new URL(service.url); if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || Number(url.port) !== service.port) throw new Error("Preview endpoint escaped its owned loopback service");
+    const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(10_000) });
+    state.initialHttpStatus = response.status; state.initialHttpBody = await response.text(); state.healthyService = service;
+    const o = await observeRuntime(ctx, state); state.queryLinks = o.links;
+    const products = [];
+    for (const id of [(state.task as Entity).id, ctx.issue()!.id]) products.push(...await ctx.input.api.get<IssueWorkProduct[]>(`/api/issues/${id}/work-products`));
+    state.previewProducts = products;
+    const source = ctx.input.page.url(); await ctx.input.page.goto(service.url, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByRole("heading", { name: String(state.marker), exact: true })).toBeVisible(); state.browserPreviewVisible = true;
+    await ctx.input.capture("real-website-preview", "The actual provider-written website is served by the ready managed service", "runtime-website-preview.png");
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+    const beforeRuns = (await ctx.allRuns()).map(run => run.id);
+    const beforeOperations = (await ctx.input.api.get<WorkspaceOperation[]>(`/api/execution-workspaces/${ws.id}/workspace-operations`)).map(row => row.id);
+    await writeFile(String(state.failureFlag), "Deliberate bounded owned process-start failure fixture", "utf8");
+    await sendChatMessage(ctx.input.page, "Restart that same website preview now and tell me whether it actually comes back. If startup fails, keep that failure visible and explain the next repair step; do not claim a working preview.");
+    const settled = await pollUntil({ label: "Explicit preview restart reports its genuine startup failure", deadlineAt: Date.now() + 180_000,
+      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), operations: await ctx.input.api.get<WorkspaceOperation[]>(`/api/execution-workspaces/${ws.id}/workspace-operations`) }; },
+      accept: value => value.source?.conversationState === "waiting" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded")
+        && value.operations.some(row => !beforeOperations.includes(row.id) && row.status === "failed") });
+    const ids = settled.runs.filter(run => !beforeRuns.includes(run.id)).map(run => run.id); state.failureRunIds = ids;
+    state.failureOperation = settled.operations.find(row => !beforeOperations.includes(row.id) && row.status === "failed" && ids.includes(row.heartbeatRunId ?? ""));
+    const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
+    state.failureReply = comments.filter(row => ids.includes(row.createdByRunId ?? "") && row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n");
+    state.failedHttpReachable = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(5_000) }).then(async res => { await res.body?.cancel(); return res.ok; }, () => false);
+  },
+  observe: observeRuntime,
+  async verifyResult(ctx, state) {
+    const source = ctx.input.page.url(); const ws = state.workspace as Entity;
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/execution-workspaces/${ws.id}`, { waitUntil: "domcontentloaded" });
+    await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByRole("heading", { name: ws.name!, exact: true }).first()).toBeVisible();
+    await expect(ctx.input.page.getByText(/failed|exit 17|startup/i).first()).toBeVisible();
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async cleanup(ctx, state) {
+    try { if (state.workspace) await ctx.input.api.post(`/api/execution-workspaces/${(state.workspace as Entity).id}/runtime-services/stop`, {}); }
+    finally { if (state.failureFlag) await unlink(String(state.failureFlag)).catch(error => { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }); }
+  },
+})];

tests/runner-e2e/chat-catalog-domains/runtime-private.test.ts

@@ -0,0 +1,12 @@
+import { createHash } from "node:crypto";
+import { it, expect } from "vitest";
+import { sshPinnedHost } from "./runtime-private.js";
+function valid() { const publicBytes = Buffer.from("synthetic public key bytes for pin calibration");
+  return { hostname: "owned-fixture.invalid", port: "2200", login: "eval", allowedRemoteRoot: "/srv/paperclip-eval-disposable",
+    knownHosts: `[owned-fixture.invalid]:2200 ssh-ed25519 ${publicBytes.toString("base64")}`,
+    hostKeySha256: `SHA256:${createHash("sha256").update(publicBytes).digest("base64").replace(/=+$/, "")}` }; }
+it("accepts only the exact provided owned host and matching known-host public-key pin", () => expect(sshPinnedHost(valid())).toBe(true));
+it("rejects a key belonging to a different host", () => { const o = valid(); o.hostname = "other.invalid"; expect(sshPinnedHost(o)).toBe(false); });
+it("rejects a mismatching public-key fingerprint", () => { const o = valid(); o.hostKeySha256 = "SHA256:wrong"; expect(sshPinnedHost(o)).toBe(false); });
+it("rejects an unscoped remote filesystem root", () => { const o = valid(); o.allowedRemoteRoot = "/"; expect(sshPinnedHost(o)).toBe(false); });
+it("rejects accepting multiple unbounded host identities", () => { const o = valid(); o.knownHosts += `\n${o.knownHosts}`; expect(sshPinnedHost(o)).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-private.ts

@@ -0,0 +1,40 @@
+import { createHash } from "node:crypto";
+import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+export type RuntimePrivateCaseId = keyof typeof runtimePrivateRequirements;
+export function runtimePrivateSpec(id: RuntimePrivateCaseId) {
+  const spec = runtimePrivateRequirements[id];
+  return { requiredCredentials: spec.credentials, externalInput: { provider: spec.provider, fields: spec.fields } };
+}
+/** Values stay in the caller's private closure, never returned as setup state. */
+export function selectedRuntimeInputs<I extends RuntimePrivateCaseId>(ctx: ChatCatalogContext, id: I) {
+  if (ctx.scenario.id !== id) throw new Error("Runtime private inputs belong to a different selected case");
+  const spec = runtimePrivateRequirements[id];
+  const config = ctx.input.privateInputs?.configuration("external-provider") as { schemaVersion?: number; providers?: Record<string, {
+    disposable?: boolean; resourceIds?: string[]; prerequisites?: Record<string, string> }> } | undefined;
+  const binding = config?.schemaVersion === 1 ? config.providers?.[spec.provider] : undefined;
+  if (!binding || binding.disposable !== true || !binding.resourceIds?.length || !binding.prerequisites
+    || spec.fields.some(field => !binding.prerequisites?.[field]))
+    throw new Error(`Setup blocker ${id}: selected disposable ${spec.provider} inputs require ${spec.fields.join(", ")}; values withheld`);
+  for (const credential of spec.credentials) if (!ctx.input.privateInputs?.credentials[credential])
+    throw new Error(`Setup blocker ${id}: missing declared selected credential ${credential}`);
+  const values = Object.fromEntries(spec.fields.map(field => [field, binding.prerequisites![field]!])) as Record<typeof spec.fields[number], string>;
+  return { values, resourceIds: binding.resourceIds, credentials: ctx.input.privateInputs!.credentials,
+    reference(name: string) { const value = ctx.input.privateInputs?.read(name); if (!value) throw new Error(`Selected runtime reference ${name} is unavailable`); return value; } };
+}
+export function sshPinnedHost(input: { hostname: string; port: string; knownHosts: string; hostKeySha256: string; allowedRemoteRoot: string; login: string }) {
+  const port = Number(input.port);
+  if (!Number.isSafeInteger(port) || port < 1 || port > 65535 || !input.hostname || /[\s\r\n]/.test(input.hostname)
+    || !/^[a-zA-Z0-9_.-]+$/.test(input.login) || !input.allowedRemoteRoot.startsWith("/")
+    || input.allowedRemoteRoot.split("/").includes("..")
+    || ["/", "/home", "/root", "/tmp", "/var", "/usr", "/etc"].includes(input.allowedRemoteRoot.replace(/\/+$/, "") || "/")) return false;
+  const host = port === 22 ? input.hostname : `[${input.hostname}]:${port}`;
+  const lines = input.knownHosts.trim().split(/\r?\n/).filter(line => line.trim() && !line.startsWith("#"));
+  return lines.length === 1 && lines.every(line => {
+    const [hosts, kind, encoded] = line.trim().split(/\s+/);
+    if (hosts !== host || !kind || !/^(?:ssh-|ecdsa-)/.test(kind) || !encoded) return false;
+    const bytes = Buffer.from(encoded, "base64"); if (!bytes.length) return false;
+    const fingerprint = `SHA256:${createHash("sha256").update(bytes).digest("base64").replace(/=+$/, "")}`;
+    return fingerprint === input.hostKeySha256;
+  });
+}

tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts

@@ -0,0 +1,17 @@
+import { it, expect } from "vitest";
+import { gradeQueuedFollowup } from "./runtime-queue.js";
+function valid() { return { scenarioId: "steering.queue-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
+  state: { task: { id: "website" }, worker: { id: "morgan" }, workerStatusAtReply: "running", canonicalRunId: "controller",
+    queuedAtReply: { issueId: "website", state: "queued", entries: [{ position: 0, comment: { id: "contact", body: "Add a contact section", authorAgentId: "ceo", createdByRunId: "controller" } }] },
+    queryReply: "I saved the contact request in the task queue for the next eligible turn.", consumedCommentId: "contact", deliveryNativeInputBound: true,
+    consumingRunCount: 1, queueEmptyAfterDelivery: true, initialBriefRevisionRetained: true, contactResultRevision: true },
+  agents: [], tasks: [{ id: "website", assigneeAgentId: "morgan", status: "done" }], links: [], activity: [], reply: "", humanQueries: [],
+  runs: [{ id: "prior", agentId: "morgan", status: "succeeded" }, { id: "controller", agentId: "ceo", status: "succeeded" }, { id: "successor", agentId: "morgan", status: "succeeded" }] }; }
+it("accepts actual ordered queue admission and one later consuming native turn", () => expect(gradeQueuedFollowup(valid()).passed).toBe(true));
+it("rejects absent queue evidence", () => expect(gradeQueuedFollowup(undefined).passed).toBe(false));
+it("rejects forwarding to a similarly named task", () => { const o = valid(); o.state.queuedAtReply.issueId = "other"; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects claiming implementation while the requirement is only queued", () => { const o = valid(); o.state.queryReply = "The contact section is complete."; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects a successor without the actual saved comment in its native input", () => { const o = valid(); o.state.deliveryNativeInputBound = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects duplicate requirement admission", () => { const o = valid(); o.state.queuedAtReply.entries.push({ ...o.state.queuedAtReply.entries[0]!, position: 1 }); expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects losing the original worker's saved work", () => { const o = valid(); o.state.initialBriefRevisionRetained = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects output attributed to the wrong native revision", () => { const o = valid(); o.state.contactResultRevision = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-queue.ts

@@ -0,0 +1,90 @@
+import { expect } from "@playwright/test";
+import { writeFile } from "node:fs/promises";
+import { pollUntil } from "../api.js";
+import { checks, type Entity } from "./internal-support.js";
+import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js";
+import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js";
+export function gradeQueuedFollowup(value: unknown) {
+  const c = checks(); const o = runtimeObservation(value);
+  c.add("real-queue-and-delivery-evidence", !!o, "The actual task queue, saved requirement, real initial worker and later consuming native input are required."); if (!o) return c.result();
+  const task = o.state.task as Entity; const queued = o.state.queuedAtReply as IssueQueuedCommentQueue | undefined;
+  const requirement = queued?.entries.find(entry => /contact/i.test(entry.comment.body));
+  c.add("correct-task-ordered-waiting-requirement-once", !!queued && queued.issueId === task.id && ["queued", "deferred"].includes(queued.state ?? "")
+    && queued.entries.length === 1 && requirement?.position === 0 && o.state.workerStatusAtReply === "running"
+    && requirement.comment.authorAgentId === o.leadId && (requirement.comment as typeof requirement.comment & { createdByRunId?: string }).createdByRunId === o.state.canonicalRunId,
+  "Exactly one contact-section instruction is durably queued in order on the genuine running Website task and attributed to the source controller turn.");
+  c.add("honest-not-consumed-yet", /queue|sav|forward|wait|next|follow.up|pending/i.test(String(o.state.queryReply))
+    && !/already (?:implemented|consumed|added)|contact section (?:is|has been) (?:ready|added|complete)|I(?:'ve| have)? implemented/i.test(String(o.state.queryReply)),
+  "The canonical answer describes pending delivery rather than pretending an active worker already consumed or implemented it.");
+  c.add("normal-continuation-consumes-exact-input-once", !!requirement && o.state.consumedCommentId === requirement.comment.id
+    && o.state.deliveryNativeInputBound === true && o.state.consumingRunCount === 1 && o.state.queueEmptyAfterDelivery === true
+    && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded")
+    && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2,
+  "The same saved requirement is consumed by one ordinary successor native turn after the real prior owner settles.");
+  c.add("earlier-work-and-contact-result-retained", o.state.initialBriefRevisionRetained === true && o.state.contactResultRevision === true
+    && o.tasks.length === 1 && o.tasks[0]?.id === task.id && o.tasks[0]?.assigneeAgentId === (o.state.worker as Entity).id && o.tasks[0]?.status === "done",
+  "The original worker's saved brief revision remains inspectable and the successor saves an actual contact-section result on the same task.");
+  return c.result();
+}
+export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, {
+  sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"],
+  expectedRunCount: 3, attemptTimeoutMs: 720_000,
+  setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] },
+  async verifySetup(ctx, state) {
+    const c = checks(); const lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
+    const worker = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${(state.worker as Entity).id}`); const run = (await ctx.allRuns()).find(row => row.id === state.workerRunId);
+    c.add("lead-ready", lead.lifecycleState === "ready" && ["active", "idle"].includes(lead.status ?? ""), "The actual controller is admitted normally.");
+    c.add("real-busy-worker", worker.lifecycleState === "ready" && ["active", "running"].includes(worker.status ?? "") && run?.status === "running" && state.workerWasReallyWaiting === true,
+    "Morgan is genuinely busy in a native tool wait, without a fabricated busy state.");
+    c.add("worker-task-scope", run?.agentId === worker.id && run.companyId === ctx.input.fixtures.company.id
+      && [run.contextSnapshot?.issueId, (run as typeof run & { nativeIssueId?: string })?.nativeIssueId].includes((state.task as Entity).id), "The actual active owner belongs to the exact task and company.");
+    return c.result();
+  },
+  async act(ctx, state) {
+    const phase = await pollUntil({ label: "Canonical follow-up saved while Morgan remains active", deadlineAt: Date.now() + 150_000,
+      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${(state.task as Entity).id}/queued-comments`) }; },
+      accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2
+        && value.runs.find(run => run.id === state.workerRunId)?.status === "running"
+        && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
+        && value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1 });
+    const canonical = phase.runs.find(run => run.agentId === ctx.input.fixtures.agent.id)!;
+    state.canonicalRunId = canonical.id; state.workerStatusAtReply = phase.runs.find(run => run.id === state.workerRunId)!.status;
+    state.queuedAtReply = phase.queue; state.queryReply = (await observeRuntime(ctx, state)).reply;
+    const source = ctx.input.page.url(), task = state.task as Entity;
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
+    const requirement = phase.queue.entries.find(entry => /contact/i.test(entry.comment.body))!;
+    await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${requirement.comment.id}`)).toBeVisible();
+    await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png");
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+    const marker = `Original website brief ${ctx.input.nonce}`; state.briefMarker = marker;
+    await writeFile((state.gate as { gate: string }).gate, `The human brief is now ready: save an initial task document with the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, "utf8");
+    const delivered = await pollUntil({ label: "Normal successor receives actual queued contact requirement", deadlineAt: Date.now() + 240_000,
+      load: async () => ({ task: await ctx.input.api.get<Entity>(`/api/issues/${task.id}`), runs: await ctx.allRuns(), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${task.id}/queued-comments`) }),
+      accept: value => value.task.status === "done" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded") && value.queue.entries.length === 0,
+      reject: value => value.runs.length > 3 || value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The single declared queued continuation failed or duplicated work" : undefined });
+    const successors = delivered.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId);
+    const inputs = await Promise.all(successors.map(run => ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`)));
+    state.consumedCommentId = requirement.comment.id; state.consumingRunCount = successors.length;
+    state.deliveryNativeInputBound = inputs.length === 1 && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.id) === true
+      && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.body) === true;
+    state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0;
+  },
+  async observe(ctx, state) {
+    const o = await observeRuntime(ctx, state);
+    state.initialBriefRevisionRetained = o.revisions.some(revision => revision.createdByRunId === state.workerRunId && (revision.body ?? "").includes(String(state.briefMarker)));
+    const successorIds = o.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId).map(run => run.id);
+    state.contactResultRevision = o.revisions.some(revision => successorIds.includes(revision.createdByRunId ?? "") && /contact/i.test(revision.body ?? "")
+      && revision.createdByAgentId === (state.worker as Entity).id && o.documents.some(doc => doc.id === revision.documentId && doc.latestRevisionId === revision.id));
+    return o;
+  },
+  async verifyResult(ctx, state) {
+    const source = ctx.input.page.url(), task = state.task as Entity;
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
+    await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible();
+    await expect(ctx.input.page.getByTestId("task-chat-queued-messages")).toHaveCount(0);
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  cleanup: async (_ctx, state) => releaseRuntimeWorker(state),
+})];

tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts

@@ -5,7 +5,7 @@ import { checks, sameTask, type Entity } from "./internal-support.js";
 import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 interface WorkspaceView { workspace: { id: string; companyId: string; projectId?: string | null; name: string; cwd: string }; repositories: unknown[] }
-export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { artifact?: boolean; workerName?: string } = {}) {
+export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { artifact?: boolean; workerName?: string; html?: boolean } = {}) {
   const { execution, fixtures, workspacePath, nonce, api } = ctx.input;
   const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id,
     workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce });
@@ -17,7 +17,7 @@ export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { art
   const marker = `Saved website prerequisite ${nonce}`;
   const previousRuns = await ctx.allRuns();
   const task = await api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
-    description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. Save a brief task document as evidence, ${options.artifact ? "register the actual website-fixture.txt file as a downloadable artifact work product titled website-fixture.txt," : ""} then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` });
+    description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. ${options.html ? `Also save index.html as a semantic HTML page with that same visible reference in an h1 and a short Acorn product paragraph. Do not start any preview server yet.` : ""} Save a brief task document as evidence, ${options.artifact ? "register the actual website-fixture.txt file as a downloadable artifact work product titled website-fixture.txt," : ""} then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` });
   await pollUntil({ label: "Real projectless website fixture completion", deadlineAt: Date.now() + 180_000,
     load: async () => ({ task: await api.get<Entity>(`/api/issues/${task.id}`), runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)) }),
     accept: value => value.task.status === "done" && value.runs.length === 1 && value.runs.every(run => run.status === "succeeded"),