diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index 3bff465a305..116d9163fe5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -206,8 +206,8 @@ }, { "scenarioId": "steering.queue-follow-up", - "readinessClass": "local", - "needs": "Real busy Morgan; save once correct task queue; order/wait state and actual later delivery." + "readinessClass": "implemented", + "needs": "Actual busy native Morgan; canonical contact instruction saved once into correct ordered task queue with source-run attribution, visible waiting UI; one normal later native input consumes exact saved comment and preserves initial document revision plus contact result." }, { "scenarioId": "steering.edit-queued-message", @@ -286,8 +286,8 @@ }, { "scenarioId": "workspaces.preview-service", - "readinessClass": "local", - "needs": "Owned ready workspace with real website files; start supported service; independent readiness HTTP/endpoint view plus startup-failure branch." + "readinessClass": "implemented", + "needs": "Actual native HTML prerequisite; preconfigured stopped managed service, canonical start, independent HTTP/content/browser/service/work-product identity, explicit actual failed restart and phase-attributed actionable error." }, { "scenarioId": "workspaces.inspect-git", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-preview.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-preview.test.ts new file mode 100644 index 00000000000..fb90cbd2084 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-preview.test.ts @@ -0,0 +1,20 @@ +import { it, expect } from "vitest"; +import { gradePreview } from "./runtime-preview.js"; +function valid() { const url = "http://127.0.0.1:43210/"; + return { scenarioId: "workspaces.preview-service", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { workspace: { id: "actual" }, task: { id: "website" }, marker: "Acorn marker", websiteBytes: "

Acorn marker

", + initialHttpBody: "

Acorn marker

", initialHttpStatus: 200, browserPreviewVisible: true, + healthyService: { id: "service", executionWorkspaceId: "actual", companyId: "company", status: "running", healthStatus: "healthy", serviceName: "Website preview", url }, + queryLinks: [url], previewProducts: [{ id: "preview", executionWorkspaceId: "actual", runtimeServiceId: "service", url, type: "preview_url", createdByRunId: "canonical" }], + failureOperation: { id: "startup-failure", status: "failed", executionWorkspaceId: "actual", heartbeatRunId: "restart" }, + failureRunIds: ["restart"], failedHttpReachable: false, failureReply: "The preview restart failed because its server process exited. Inspect the startup error and repair it before starting again." }, + agents: [], tasks: [{ id: "website" }], links: [], activity: [], humanQueries: [], reply: "", + runs: [{ id: "website-setup", agentId: "morgan", status: "succeeded" }, { id: "canonical", agentId: "ceo", status: "succeeded" }, { id: "restart", agentId: "ceo", status: "succeeded" }] }; } +it("accepts actual ready HTTP preview then a separately observed failed restart", () => expect(gradePreview(valid()).passed).toBe(true)); +it("rejects missing real preview evidence", () => expect(gradePreview(undefined).passed).toBe(false)); +it("rejects a service record without HTTP content readiness", () => { const o = valid(); o.state.initialHttpStatus = 503; expect(gradePreview(o).passed).toBe(false); }); +it("rejects serving another website's bytes", () => { const o = valid(); o.state.initialHttpBody = "Other website"; expect(gradePreview(o).passed).toBe(false); }); +it("rejects a similarly named service from another workspace", () => { const o = valid(); o.state.healthyService.executionWorkspaceId = "other"; expect(gradePreview(o).passed).toBe(false); }); +it("rejects a preview product from an unrelated run", () => { const o = valid(); o.state.previewProducts[0]!.createdByRunId = "foreign"; expect(gradePreview(o).passed).toBe(false); }); +it("rejects claiming readiness after the actual startup failure", () => { const o = valid(); o.state.failureReply = "The preview is now ready."; expect(gradePreview(o).passed).toBe(false); }); +it("rejects failure attributed to the initial healthy turn", () => { const o = valid(); o.state.failureOperation.heartbeatRunId = "canonical"; expect(gradePreview(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts b/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts new file mode 100644 index 00000000000..5aeb8ff33fc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts @@ -0,0 +1,106 @@ +import { expect } from "@playwright/test"; +import { readFile, realpath, writeFile, unlink } from "node:fs/promises"; +import path from "node:path"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { savedTaskWorkspace } from "./runtime-workspaces.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import type { WorkspaceRuntimeService } from "../../../packages/shared/src/types/workspace-runtime.js"; +import type { WorkspaceOperation } from "../../../packages/shared/src/types/workspace-operation.js"; +import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-product.js"; +interface PreviewWorkspace extends Entity { cwd: string; config: Record; runtimeServices?: WorkspaceRuntimeService[] } +export function gradePreview(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-preview-lifecycle-evidence", !!o, "Actual service identity, HTTP content, source output and subsequent startup failure must be independently observed."); if (!o) return c.result(); + const service = o.state.healthyService as WorkspaceRuntimeService | undefined; const workspace = o.state.workspace as PreviewWorkspace; + const products = o.state.previewProducts as IssueWorkProduct[] | undefined; + c.add("real-ready-service-correct-workspace", !!service && service.executionWorkspaceId === workspace.id && service.companyId === o.companyId + && service.status === "running" && service.healthStatus === "healthy" && service.serviceName === "Website preview" + && o.state.initialHttpStatus === 200 && o.state.initialHttpBody === o.state.websiteBytes + && typeof o.state.initialHttpBody === "string" && o.state.initialHttpBody.includes(String(o.state.marker)) && o.state.browserPreviewVisible === true, + "A real managed Website preview serves the actual provider-written website bytes in the intended workspace and is usable in the browser."); + c.add("accessible-registered-preview", !!service?.url && Array.isArray(products) && products.some(row => row.executionWorkspaceId === workspace.id + && row.runtimeServiceId === service.id && row.url === service.url && ["preview_url", "runtime_service"].includes(row.type) + && o.runs.some(run => run.id === row.createdByRunId && run.agentId === o.leadId)) + && (o.state.queryLinks as string[] | undefined)?.includes(service.url) === true, + "The actual canonical reply exposes the ready endpoint and its run-attributed work product is tied to this exact workspace/service."); + const failure = o.state.failureOperation as WorkspaceOperation | undefined; const reply = String(o.state.failureReply ?? ""); + c.add("startup-failure-exposed-actionably", !!failure && failure.status === "failed" && failure.executionWorkspaceId === workspace.id + && (o.state.failureRunIds as string[] | undefined)?.includes(failure.heartbeatRunId ?? "") === true + && o.state.failedHttpReachable === false && /fail|error|could not|couldn't|did not start|not running/i.test(reply) + && /start|process|server|preview|exit/i.test(reply) && !/preview (?:is|has) (?:now )?(?:ready|running)|successfully restart/i.test(reply), + "A separate explicit restart really fails under the fixture's process-start fault; its own answer preserves the failure and repair context."); + c.add("no-unrelated-work-or-publication", o.tasks.length === 1 && o.tasks[0]?.id === (o.state.task as Entity).id + && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded"), + "The prepared website plus two explicit controller turns are the only native work; no remote deployment or extra task is substituted."); + return c.result(); +} +export const runtimePreviewCases = [runtimeDefinition("workspaces.preview-service", async ctx => { + if (ctx.input.execution.environment.id !== "local") throw new Error("Owned loopback website preview fixture requires the actual local target"); + const state = await savedTaskWorkspace(ctx, { html: true }); const cwd = await realpath(state.workspaceBefore.workspace.cwd); + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!root || !cwd.startsWith(`${await realpath(root)}${path.sep}`)) throw new Error("Preview fixture escaped the isolated task directory"); + const websiteBytes = await readFile(path.join(cwd, "index.html"), "utf8"); + if (!/{res.writeHead(200,{'content-type':'text/html; charset=utf-8'});res.end(body)}).listen(Number(process.argv[2]),'127.0.0.1');`, "utf8"); + const workspace = await ctx.input.api.get(`/api/execution-workspaces/${state.workspaceBefore.workspace.id}`); + await ctx.input.api.patch(`/api/execution-workspaces/${workspace.id}`, { config: { ...workspace.config, desiredState: "stopped", workspaceRuntime: { services: [{ + name: "Website preview", command: "node preview-server.cjs {{port}}", cwd: ".", port: { type: "auto" }, lifecycle: "shared", stopPolicy: { type: "manual" }, + readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 100 }, + expose: { type: "url", urlTemplate: "http://127.0.0.1:{{port}}" }, + }] } } }); + const saved = await ctx.input.api.get(`/api/execution-workspaces/${workspace.id}`); + if (saved.runtimeServices?.some(row => row.status === "running")) throw new Error("Preview setup must not start the target service"); + return { ...state, workspace: saved, websiteBytes, failureFlag }; +}, gradePreview, { + sourceFiles: ["chat-catalog-domains/runtime-preview.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml"], + expectedHumanMessageCount: 2, expectedRunCount: 3, attemptTimeoutMs: 720_000, + async act(ctx, state) { + await pollUntil({ label: "Actual canonical preview service readiness", deadlineAt: Date.now() + 180_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), workspace: await ctx.input.api.get(`/api/execution-workspaces/${(state.workspace as Entity).id}`) }; }, + accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") + && value.workspace.runtimeServices?.some(service => service.status === "running" && service.healthStatus === "healthy") === true }); + const ws = await ctx.input.api.get(`/api/execution-workspaces/${(state.workspace as Entity).id}`); + const service = ws.runtimeServices?.find(row => row.serviceName === "Website preview" && row.status === "running")!; + if (!service?.url) throw new Error("A created service record without an accessible ready endpoint is not a preview"); + const url = new URL(service.url); if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || Number(url.port) !== service.port) throw new Error("Preview endpoint escaped its owned loopback service"); + const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(10_000) }); + state.initialHttpStatus = response.status; state.initialHttpBody = await response.text(); state.healthyService = service; + const o = await observeRuntime(ctx, state); state.queryLinks = o.links; + const products = []; + for (const id of [(state.task as Entity).id, ctx.issue()!.id]) products.push(...await ctx.input.api.get(`/api/issues/${id}/work-products`)); + state.previewProducts = products; + const source = ctx.input.page.url(); await ctx.input.page.goto(service.url, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: String(state.marker), exact: true })).toBeVisible(); state.browserPreviewVisible = true; + await ctx.input.capture("real-website-preview", "The actual provider-written website is served by the ready managed service", "runtime-website-preview.png"); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const beforeRuns = (await ctx.allRuns()).map(run => run.id); + const beforeOperations = (await ctx.input.api.get(`/api/execution-workspaces/${ws.id}/workspace-operations`)).map(row => row.id); + await writeFile(String(state.failureFlag), "Deliberate bounded owned process-start failure fixture", "utf8"); + await sendChatMessage(ctx.input.page, "Restart that same website preview now and tell me whether it actually comes back. If startup fails, keep that failure visible and explain the next repair step; do not claim a working preview."); + const settled = await pollUntil({ label: "Explicit preview restart reports its genuine startup failure", deadlineAt: Date.now() + 180_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), operations: await ctx.input.api.get(`/api/execution-workspaces/${ws.id}/workspace-operations`) }; }, + accept: value => value.source?.conversationState === "waiting" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded") + && value.operations.some(row => !beforeOperations.includes(row.id) && row.status === "failed") }); + const ids = settled.runs.filter(run => !beforeRuns.includes(run.id)).map(run => run.id); state.failureRunIds = ids; + state.failureOperation = settled.operations.find(row => !beforeOperations.includes(row.id) && row.status === "failed" && ids.includes(row.heartbeatRunId ?? "")); + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.failureReply = comments.filter(row => ids.includes(row.createdByRunId ?? "") && row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n"); + state.failedHttpReachable = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(5_000) }).then(async res => { await res.body?.cancel(); return res.ok; }, () => false); + }, + observe: observeRuntime, + async verifyResult(ctx, state) { + const source = ctx.input.page.url(); const ws = state.workspace as Entity; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/execution-workspaces/${ws.id}`, { waitUntil: "domcontentloaded" }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: ws.name!, exact: true }).first()).toBeVisible(); + await expect(ctx.input.page.getByText(/failed|exit 17|startup/i).first()).toBeVisible(); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async cleanup(ctx, state) { + try { if (state.workspace) await ctx.input.api.post(`/api/execution-workspaces/${(state.workspace as Entity).id}/runtime-services/stop`, {}); } + finally { if (state.failureFlag) await unlink(String(state.failureFlag)).catch(error => { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }); } + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-private.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-private.test.ts new file mode 100644 index 00000000000..75912023d55 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-private.test.ts @@ -0,0 +1,12 @@ +import { createHash } from "node:crypto"; +import { it, expect } from "vitest"; +import { sshPinnedHost } from "./runtime-private.js"; +function valid() { const publicBytes = Buffer.from("synthetic public key bytes for pin calibration"); + return { hostname: "owned-fixture.invalid", port: "2200", login: "eval", allowedRemoteRoot: "/srv/paperclip-eval-disposable", + knownHosts: `[owned-fixture.invalid]:2200 ssh-ed25519 ${publicBytes.toString("base64")}`, + hostKeySha256: `SHA256:${createHash("sha256").update(publicBytes).digest("base64").replace(/=+$/, "")}` }; } +it("accepts only the exact provided owned host and matching known-host public-key pin", () => expect(sshPinnedHost(valid())).toBe(true)); +it("rejects a key belonging to a different host", () => { const o = valid(); o.hostname = "other.invalid"; expect(sshPinnedHost(o)).toBe(false); }); +it("rejects a mismatching public-key fingerprint", () => { const o = valid(); o.hostKeySha256 = "SHA256:wrong"; expect(sshPinnedHost(o)).toBe(false); }); +it("rejects an unscoped remote filesystem root", () => { const o = valid(); o.allowedRemoteRoot = "/"; expect(sshPinnedHost(o)).toBe(false); }); +it("rejects accepting multiple unbounded host identities", () => { const o = valid(); o.knownHosts += `\n${o.knownHosts}`; expect(sshPinnedHost(o)).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-private.ts b/tests/runner-e2e/chat-catalog-domains/runtime-private.ts new file mode 100644 index 00000000000..a59547d0a7c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-private.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +export type RuntimePrivateCaseId = keyof typeof runtimePrivateRequirements; +export function runtimePrivateSpec(id: RuntimePrivateCaseId) { + const spec = runtimePrivateRequirements[id]; + return { requiredCredentials: spec.credentials, externalInput: { provider: spec.provider, fields: spec.fields } }; +} +/** Values stay in the caller's private closure, never returned as setup state. */ +export function selectedRuntimeInputs(ctx: ChatCatalogContext, id: I) { + if (ctx.scenario.id !== id) throw new Error("Runtime private inputs belong to a different selected case"); + const spec = runtimePrivateRequirements[id]; + const config = ctx.input.privateInputs?.configuration("external-provider") as { schemaVersion?: number; providers?: Record }> } | undefined; + const binding = config?.schemaVersion === 1 ? config.providers?.[spec.provider] : undefined; + if (!binding || binding.disposable !== true || !binding.resourceIds?.length || !binding.prerequisites + || spec.fields.some(field => !binding.prerequisites?.[field])) + throw new Error(`Setup blocker ${id}: selected disposable ${spec.provider} inputs require ${spec.fields.join(", ")}; values withheld`); + for (const credential of spec.credentials) if (!ctx.input.privateInputs?.credentials[credential]) + throw new Error(`Setup blocker ${id}: missing declared selected credential ${credential}`); + const values = Object.fromEntries(spec.fields.map(field => [field, binding.prerequisites![field]!])) as Record; + return { values, resourceIds: binding.resourceIds, credentials: ctx.input.privateInputs!.credentials, + reference(name: string) { const value = ctx.input.privateInputs?.read(name); if (!value) throw new Error(`Selected runtime reference ${name} is unavailable`); return value; } }; +} +export function sshPinnedHost(input: { hostname: string; port: string; knownHosts: string; hostKeySha256: string; allowedRemoteRoot: string; login: string }) { + const port = Number(input.port); + if (!Number.isSafeInteger(port) || port < 1 || port > 65535 || !input.hostname || /[\s\r\n]/.test(input.hostname) + || !/^[a-zA-Z0-9_.-]+$/.test(input.login) || !input.allowedRemoteRoot.startsWith("/") + || input.allowedRemoteRoot.split("/").includes("..") + || ["/", "/home", "/root", "/tmp", "/var", "/usr", "/etc"].includes(input.allowedRemoteRoot.replace(/\/+$/, "") || "/")) return false; + const host = port === 22 ? input.hostname : `[${input.hostname}]:${port}`; + const lines = input.knownHosts.trim().split(/\r?\n/).filter(line => line.trim() && !line.startsWith("#")); + return lines.length === 1 && lines.every(line => { + const [hosts, kind, encoded] = line.trim().split(/\s+/); + if (hosts !== host || !kind || !/^(?:ssh-|ecdsa-)/.test(kind) || !encoded) return false; + const bytes = Buffer.from(encoded, "base64"); if (!bytes.length) return false; + const fingerprint = `SHA256:${createHash("sha256").update(bytes).digest("base64").replace(/=+$/, "")}`; + return fingerprint === input.hostKeySha256; + }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts new file mode 100644 index 00000000000..4faca3a4ca9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -0,0 +1,17 @@ +import { it, expect } from "vitest"; +import { gradeQueuedFollowup } from "./runtime-queue.js"; +function valid() { return { scenarioId: "steering.queue-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { task: { id: "website" }, worker: { id: "morgan" }, workerStatusAtReply: "running", canonicalRunId: "controller", + queuedAtReply: { issueId: "website", state: "queued", entries: [{ position: 0, comment: { id: "contact", body: "Add a contact section", authorAgentId: "ceo", createdByRunId: "controller" } }] }, + queryReply: "I saved the contact request in the task queue for the next eligible turn.", consumedCommentId: "contact", deliveryNativeInputBound: true, + consumingRunCount: 1, queueEmptyAfterDelivery: true, initialBriefRevisionRetained: true, contactResultRevision: true }, + agents: [], tasks: [{ id: "website", assigneeAgentId: "morgan", status: "done" }], links: [], activity: [], reply: "", humanQueries: [], + runs: [{ id: "prior", agentId: "morgan", status: "succeeded" }, { id: "controller", agentId: "ceo", status: "succeeded" }, { id: "successor", agentId: "morgan", status: "succeeded" }] }; } +it("accepts actual ordered queue admission and one later consuming native turn", () => expect(gradeQueuedFollowup(valid()).passed).toBe(true)); +it("rejects absent queue evidence", () => expect(gradeQueuedFollowup(undefined).passed).toBe(false)); +it("rejects forwarding to a similarly named task", () => { const o = valid(); o.state.queuedAtReply.issueId = "other"; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects claiming implementation while the requirement is only queued", () => { const o = valid(); o.state.queryReply = "The contact section is complete."; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects a successor without the actual saved comment in its native input", () => { const o = valid(); o.state.deliveryNativeInputBound = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects duplicate requirement admission", () => { const o = valid(); o.state.queuedAtReply.entries.push({ ...o.state.queuedAtReply.entries[0]!, position: 1 }); expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects losing the original worker's saved work", () => { const o = valid(); o.state.initialBriefRevisionRetained = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects output attributed to the wrong native revision", () => { const o = valid(); o.state.contactResultRevision = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts new file mode 100644 index 00000000000..0655bf28f23 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -0,0 +1,90 @@ +import { expect } from "@playwright/test"; +import { writeFile } from "node:fs/promises"; +import { pollUntil } from "../api.js"; +import { checks, type Entity } from "./internal-support.js"; +import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; +import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js"; +export function gradeQueuedFollowup(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("real-queue-and-delivery-evidence", !!o, "The actual task queue, saved requirement, real initial worker and later consuming native input are required."); if (!o) return c.result(); + const task = o.state.task as Entity; const queued = o.state.queuedAtReply as IssueQueuedCommentQueue | undefined; + const requirement = queued?.entries.find(entry => /contact/i.test(entry.comment.body)); + c.add("correct-task-ordered-waiting-requirement-once", !!queued && queued.issueId === task.id && ["queued", "deferred"].includes(queued.state ?? "") + && queued.entries.length === 1 && requirement?.position === 0 && o.state.workerStatusAtReply === "running" + && requirement.comment.authorAgentId === o.leadId && (requirement.comment as typeof requirement.comment & { createdByRunId?: string }).createdByRunId === o.state.canonicalRunId, + "Exactly one contact-section instruction is durably queued in order on the genuine running Website task and attributed to the source controller turn."); + c.add("honest-not-consumed-yet", /queue|sav|forward|wait|next|follow.up|pending/i.test(String(o.state.queryReply)) + && !/already (?:implemented|consumed|added)|contact section (?:is|has been) (?:ready|added|complete)|I(?:'ve| have)? implemented/i.test(String(o.state.queryReply)), + "The canonical answer describes pending delivery rather than pretending an active worker already consumed or implemented it."); + c.add("normal-continuation-consumes-exact-input-once", !!requirement && o.state.consumedCommentId === requirement.comment.id + && o.state.deliveryNativeInputBound === true && o.state.consumingRunCount === 1 && o.state.queueEmptyAfterDelivery === true + && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded") + && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2, + "The same saved requirement is consumed by one ordinary successor native turn after the real prior owner settles."); + c.add("earlier-work-and-contact-result-retained", o.state.initialBriefRevisionRetained === true && o.state.contactResultRevision === true + && o.tasks.length === 1 && o.tasks[0]?.id === task.id && o.tasks[0]?.assigneeAgentId === (o.state.worker as Entity).id && o.tasks[0]?.status === "done", + "The original worker's saved brief revision remains inspectable and the successor saves an actual contact-section result on the same task."); + return c.result(); +} +export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, { + sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], + expectedRunCount: 3, attemptTimeoutMs: 720_000, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, + async verifySetup(ctx, state) { + const c = checks(); const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const worker = await ctx.input.api.get(`/api/agents/${(state.worker as Entity).id}`); const run = (await ctx.allRuns()).find(row => row.id === state.workerRunId); + c.add("lead-ready", lead.lifecycleState === "ready" && ["active", "idle"].includes(lead.status ?? ""), "The actual controller is admitted normally."); + c.add("real-busy-worker", worker.lifecycleState === "ready" && ["active", "running"].includes(worker.status ?? "") && run?.status === "running" && state.workerWasReallyWaiting === true, + "Morgan is genuinely busy in a native tool wait, without a fabricated busy state."); + c.add("worker-task-scope", run?.agentId === worker.id && run.companyId === ctx.input.fixtures.company.id + && [run.contextSnapshot?.issueId, (run as typeof run & { nativeIssueId?: string })?.nativeIssueId].includes((state.task as Entity).id), "The actual active owner belongs to the exact task and company."); + return c.result(); + }, + async act(ctx, state) { + const phase = await pollUntil({ label: "Canonical follow-up saved while Morgan remains active", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), queue: await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}/queued-comments`) }; }, + accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 + && value.runs.find(run => run.id === state.workerRunId)?.status === "running" + && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") + && value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1 }); + const canonical = phase.runs.find(run => run.agentId === ctx.input.fixtures.agent.id)!; + state.canonicalRunId = canonical.id; state.workerStatusAtReply = phase.runs.find(run => run.id === state.workerRunId)!.status; + state.queuedAtReply = phase.queue; state.queryReply = (await observeRuntime(ctx, state)).reply; + const source = ctx.input.page.url(), task = state.task as Entity; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + const requirement = phase.queue.entries.find(entry => /contact/i.test(entry.comment.body))!; + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${requirement.comment.id}`)).toBeVisible(); + await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png"); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const marker = `Original website brief ${ctx.input.nonce}`; state.briefMarker = marker; + await writeFile((state.gate as { gate: string }).gate, `The human brief is now ready: save an initial task document with the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, "utf8"); + const delivered = await pollUntil({ label: "Normal successor receives actual queued contact requirement", deadlineAt: Date.now() + 240_000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns(), queue: await ctx.input.api.get(`/api/issues/${task.id}/queued-comments`) }), + accept: value => value.task.status === "done" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded") && value.queue.entries.length === 0, + reject: value => value.runs.length > 3 || value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The single declared queued continuation failed or duplicated work" : undefined }); + const successors = delivered.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId); + const inputs = await Promise.all(successors.map(run => ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`))); + state.consumedCommentId = requirement.comment.id; state.consumingRunCount = successors.length; + state.deliveryNativeInputBound = inputs.length === 1 && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.id) === true + && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.body) === true; + state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0; + }, + async observe(ctx, state) { + const o = await observeRuntime(ctx, state); + state.initialBriefRevisionRetained = o.revisions.some(revision => revision.createdByRunId === state.workerRunId && (revision.body ?? "").includes(String(state.briefMarker))); + const successorIds = o.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId).map(run => run.id); + state.contactResultRevision = o.revisions.some(revision => successorIds.includes(revision.createdByRunId ?? "") && /contact/i.test(revision.body ?? "") + && revision.createdByAgentId === (state.worker as Entity).id && o.documents.some(doc => doc.id === revision.documentId && doc.latestRevisionId === revision.id)); + return o; + }, + async verifyResult(ctx, state) { + const source = ctx.input.page.url(), task = state.task as Entity; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-queued-messages")).toHaveCount(0); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + cleanup: async (_ctx, state) => releaseRuntimeWorker(state), +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts index 5e5db9013a2..57057e19b68 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts @@ -5,7 +5,7 @@ import { checks, sameTask, type Entity } from "./internal-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface WorkspaceView { workspace: { id: string; companyId: string; projectId?: string | null; name: string; cwd: string }; repositories: unknown[] } -export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { artifact?: boolean; workerName?: string } = {}) { +export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { artifact?: boolean; workerName?: string; html?: boolean } = {}) { const { execution, fixtures, workspacePath, nonce, api } = ctx.input; const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce }); @@ -17,7 +17,7 @@ export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { art const marker = `Saved website prerequisite ${nonce}`; const previousRuns = await ctx.allRuns(); const task = await api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, - description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. Save a brief task document as evidence, ${options.artifact ? "register the actual website-fixture.txt file as a downloadable artifact work product titled website-fixture.txt," : ""} then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` }); + description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. ${options.html ? `Also save index.html as a semantic HTML page with that same visible reference in an h1 and a short Acorn product paragraph. Do not start any preview server yet.` : ""} Save a brief task document as evidence, ${options.artifact ? "register the actual website-fixture.txt file as a downloadable artifact work product titled website-fixture.txt," : ""} then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` }); await pollUntil({ label: "Real projectless website fixture completion", deadlineAt: Date.now() + 180_000, load: async () => ({ task: await api.get(`/api/issues/${task.id}`), runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)) }), accept: value => value.task.status === "done" && value.runs.length === 1 && value.runs.every(run => run.status === "succeeded"),