import { expect } from "@playwright/test"; import { writeFile } from "node:fs/promises"; import type { ActivityEvent } from "../../../packages/shared/src/types/activity.js"; import type { IssueQueuedCommentQueue, IssueThreadInteraction } from "../../../packages/shared/src/types/issue.js"; import { pollUntil } from "../api.js"; import { submitTaskReply } from "../user-actions.js"; import { checks, type Entity } from "./internal-support.js"; import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; export function acornNamingDirection(reply: string) { if (/\b(?:not|instead of|rather than|avoid|reject)\s+(?:the name\s+)?Acorn\b|\bAcorn\s+(?:is|was|would be)\s+(?:the\s+)?(?:wrong|incorrect|invalid|rejected)\b/i.test(reply)) return false; const selectedNames = [...reply.matchAll(/\b(?:[Uu]se|[Cc]hoose|[Nn]ame it|[Cc]all it|[Aa]nswer with)\s+["']?([A-Z][a-zA-Z0-9-]*)/g)].filter(match => !/\b(?:not|don't|avoid)\s*$/i.test(reply.slice(Math.max(0, (match.index ?? 0) - 12), match.index))).map(match => match[1]!); if (selectedNames.some(name => !["Acorn", "The", "Submit", "That", "Its", "This"].includes(name))) return false; return /\bAcorn\b/i.test(reply) && /question|answer|respond|decision|nam(?:e|ing)/i.test(reply); } export function ordinaryApprovalEntries(queue: Pick) { return queue.entries.filter(row => !row.source); } export function gradeApprovalQueue(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result(); const s = o.state, question = s.questionBefore as IssueThreadInteraction, answered = s.questionAfter as IssueThreadInteraction; c.add("genuine-prior-worker-question-and-saved-comments", !!question && question.kind === "ask_user_questions" && question.status === "pending" && question.issueId === (s.task as Entity).id && question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.workerRunId && s.workerWasReallyWaiting === true && s.ordinaryQueueOwned === true && s.ordinaryQueuedBeforeAnswer === true, "Morgan's actual native turn asks the naming question and remains active while the real human's separate ordinary comments are pending."); const results = answered?.result as { answers?: Array<{ questionId?: string; otherText?: string }> } | null; c.add("actual-bound-human-answer-preserved", answered?.id === question.id && answered.status === "answered" && ((answered.resolvedByUserId === s.humanId && !answered.resolvedByAgentId && s.humanAnswerSubmitted === true) || (answered.resolvedByAgentId === o.leadId && s.canonicalAnswerAttributed === true)) && JSON.stringify(answered.payload) === JSON.stringify(question.payload) && results?.answers?.some(row => row.questionId === "name" && row.otherText === "Acorn") === true && s.responseRetainedAcrossOrdinaryMutations === true && s.responseUneditable === true, "Acorn remains the same immutable typed interaction response, distinct from the editable/deletable ordinary comment queue."); c.add("human-guidance-and-real-ordinary-mutations", acornNamingDirection(String(s.queryReply)) && (s.canonicalAnswerAttributed === true || !/I(?:'ve| have)? (?:already )?(?:answered|submitted|accepted)/i.test(String(s.queryReply))) && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true && (s.humanAnswerSubmitted === true || s.canonicalAnswerAttributed === true), "Canonical chat identifies the intended answer and actual human response control; editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); const response = s.responseQueue as IssueQueuedCommentQueue; c.add("response-own-delivery-semantics-and-once-only-admission", response?.entries.length === 1 && response.entries[0]?.source?.kind === "interaction" && response.entries[0].source.interactionId === question.id && response.entries[0].canEdit === false && response.entries[0].canDiscard === false && s.consumingRunCount === 1 && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true && o.runs.length === 3 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, "After the original owner settles, exactly one eligible native continuation receives the bound answer and saves actual Acorn work without replaying or losing it."); return c.result(); } export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => { const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }), root = `/api/companies/${ctx.input.fixtures.company.id}`; const task = await ctx.input.api.post(`${root}/issues`, { title: "Website naming task", status: "todo", assigneeAgentId: prepared.worker.id, description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run ${prepared.gate.command} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` }); const waiting = await pollUntil({ label: "Real native naming question while worker waits in its own tool process", deadlineAt: Date.now() + 180_000, load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await ctx.input.api.get(`/api/issues/${task.id}/interactions`) }), accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.runs[0].status === "running" && v.ready === "waiting" && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id, reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "succeeded", "timed_out"].includes(row.status)) ? "The actual question fixture did not reach its running pending-input boundary" : undefined }); const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(form).toBeVisible(); await form.getByRole("button", { name: /^Dismiss / }).click(); const messages = ["Preserve the original layout work.", "Use accessible labels."]; for (const body of messages) await submitTaskReply(ctx.input.page, body); const queue = await pollUntil({ label: "Actual ordinary human comments remain separate behind running Morgan", deadlineAt: Date.now() + 20_000, load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), accept: q => q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); if (queue.entries.some(row => row.comment.authorUserId !== session.user.id || !row.canEdit || !row.canDiscard)) throw new Error("Ordinary queue is not actually owned by the signed-in human"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { worker: prepared.worker, gate: prepared.gate, task, workerRunId: waiting.runs[0]!.id, workerWasReallyWaiting: true, questionBefore: waiting.questions[0]!, ordinaryQueueBefore: queue, humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true }; }, gradeApprovalQueue, { setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); state.queryReply = (await observeRuntime(ctx, state)).reply; const task = state.task as Entity, question = state.questionBefore as IssueThreadInteraction, source = ctx.input.page.url(), endpoint = `/api/issues/${task.id}/queued-comments`; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const current = (await ctx.input.api.get(`/api/issues/${task.id}/interactions`)).find(row => row.id === question.id); if (current?.status === "answered") { const canonical = canonicalPhase.runs.find(row => row.agentId === ctx.input.fixtures.agent.id)!; const activity = await ctx.input.api.get(`/api/issues/${task.id}/activity`); state.canonicalAnswerAttributed = current.resolvedByAgentId === ctx.input.fixtures.agent.id && activity.some(row => row.action === "issue.thread_interaction_answered" && row.agentId === ctx.input.fixtures.agent.id && row.runId === canonical.id && row.details?.interactionId === question.id); if (!state.canonicalAnswerAttributed) throw new Error("The existing answer was not authorized by this actual canonical native source run"); } else { await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); const reopen = ctx.input.page.getByRole("button", { name: /answer|respond|question/i }).filter({ hasText: /answer|respond/i }); if (!(await ctx.input.page.getByTestId("task-chat-composer-takeover").isVisible())) await reopen.first().click(); const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(form).toBeVisible(); await form.getByTestId("question-text-answer-composer").locator('[contenteditable="true"],textarea').first().fill("Acorn"); await form.getByRole("button", { name: /^(Submit|Send) answers$/ }).click(); state.humanAnswerSubmitted = true; } const answered = await pollUntil({ label: "Same real human naming interaction receives Acorn exactly once", deadlineAt: Date.now() + 20_000, load: () => ctx.input.api.get(`/api/issues/${task.id}/interactions`), accept: rows => rows.find(row => row.id === question.id)?.status === "answered" }); state.questionAfter = answered.find(row => row.id === question.id)!; // Mutate only ordinary comments. The interaction response is server-owned // and remains independently read after every mutation. let retained = true; async function verifyResponse() { const row = (await ctx.input.api.get(`/api/issues/${task.id}/interactions`)).find(row => row.id === question.id); retained &&= row?.status === "answered" && JSON.stringify(row.result) === JSON.stringify((state.questionAfter as IssueThreadInteraction).result); } let queue = await ctx.input.api.get(endpoint); const ordinary = ordinaryApprovalEntries(queue); if (ordinary.length !== 2 || !queue.queueId) throw new Error("The bound answer replaced rather than remained separate from the ordinary waiting queue"); const target = ordinary[0]!, edited = `${target.comment.body} Retain the original layout note.`; await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await ctx.input.page.getByTestId(`task-chat-queued-message-${target.comment.id}`).getByRole("button", { name: /^Queued message actions:/ }).click(); await ctx.input.page.getByRole("menuitem", { name: "Edit message", exact: true }).click(); await ctx.input.page.getByTestId("task-chat-composer-input").last().locator('[contenteditable="true"],textarea').first().fill(edited); await ctx.input.page.getByRole("button", { name: "Save queued message", exact: true }).click(); queue = await pollUntil({ label: "Actual ordinary full-text edit persists independently of the accepted response", deadlineAt: Date.now() + 15_000, load: () => ctx.input.api.get(endpoint), accept: q => q.entries.some(row => row.comment.id === target.comment.id && row.comment.body === edited) }); state.actualOrdinaryEdit = true; await verifyResponse(); const order = ordinaryApprovalEntries(queue).map(row => row.comment.id).reverse(), moved = ordinaryApprovalEntries(queue)[1]!, wholeOrderBefore = queue.entries.map(row => row.comment.id); const mixed = queue.entries.some(row => row.source?.kind === "interaction"); const handle = ctx.input.page.getByRole("button", { name: `Reorder queued message: ${moved.comment.body}`, exact: true }); await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space"); queue = await pollUntil({ label: "Real keyboard queue reorder persists without replaying the decision", deadlineAt: Date.now() + 15_000, load: () => ctx.input.api.get(endpoint), accept: q => mixed ? q.entries.map(row => row.comment.id).join() === wholeOrderBefore.join() : ordinaryApprovalEntries(q).map(row => row.comment.id).join() === order.join() }); state.actualOrdinaryReorder = !mixed; state.ordinaryReorderRestricted = mixed && queue.entries.map(row => row.comment.id).join() === wholeOrderBefore.join(); await verifyResponse(); for (const id of order) { await ctx.input.page.getByTestId(`task-chat-queued-discard-${id}`).click(); await pollUntil({ label: "Supported human discard removes only this ordinary comment", deadlineAt: Date.now() + 15_000, load: () => ctx.input.api.get(endpoint), accept: q => !q.entries.some(row => row.comment.id === id) }); await verifyResponse(); } state.actualOrdinaryDiscard = true; state.responseRetainedAcrossOrdinaryMutations = retained; const response = await pollUntil({ label: "Typed immutable naming answer remains queued after ordinary comments are removed", deadlineAt: Date.now() + 20_000, load: () => ctx.input.api.get(endpoint), accept: q => q.entries.length === 1 && q.entries[0]?.source?.interactionId === question.id }); state.responseQueue = response; const immutable = await ctx.input.page.request.patch(`${endpoint}/${question.id}`, { data: { queueId: response.queueId, revision: response.revision, body: "Replace the accepted name with an unauthorized comment edit" } }); state.responseUneditable = [403, 409].includes(immutable.status()); await immutable.dispose(); await verifyResponse(); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${question.id}`)).toBeVisible(); await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled(); await ctx.input.capture("actual-immutable-answer-queue", "Bound Acorn answer remains distinct from edited, reordered and removed ordinary comments", "runtime-approval-queue.png"); state.initialMarker = `Earlier website work ${ctx.input.nonce}`; await writeFile((state.gate as { gate: string }).gate, `The current pending-input turn is settling. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; const full = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { interactionId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`); state.responseNativeInputBound = full?.contextSnapshot?.interactionId === question.id && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(question.id) === true && full.runnerProfileJson.nativeExecutionInput.task.prompt.includes("Acorn"); state.queueEmptyAfterDelivery = done.queue.entries.length === 0; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, async observe(ctx, state) { const o = await observeRuntime(ctx, state); const question = state.questionBefore as IssueThreadInteraction; state.questionAfter = (await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}/interactions`)).find(row => row.id === question.id); state.actualAcornResult = o.revisions.some(row => row.createdByRunId === state.consumerRunId && row.createdByAgentId === (state.worker as Entity).id && /Acorn/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); state.initialWorkRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && (row.body ?? "").includes(String(state.initialMarker))); return o; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible(); await expect(ctx.input.page.getByTestId("task-chat-queued-messages")).toHaveCount(0); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, cleanup: async (_ctx, state) => releaseRuntimeWorker(state), })];