From 7d06729d056661d6cdae53620aaae29fb94df689 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:20:07 -0600 Subject: [PATCH] fix(evals): gate steering.approval-delivery on the parked pending-input turn A native worker that saves a durable wake-based question parks its turn: the run succeeds with a yielded disposition whose continuation is the question's response wake. The fixture waited for that run to stay running, which the product never does, so setup failed before the canonical query. Gate the question on that saved yield instead, then have an ordinary human message wake a real second Morgan turn that waits at the task gate. Ordinary comments and the bound answer queue behind that active turn; one continuation after it receives the answer. Replace the reply keyword checks with the saved answer path (human response control or attributed canonical response). Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 4 +- .../runtime-approval-queue.test.ts | 32 +++++--- .../runtime-approval-queue.ts | 75 ++++++++++++------- .../runtime-approval-workspace.test.ts | 37 ++++++--- 4 files changed, 100 insertions(+), 48 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..b49ec36fcc9 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5440,9 +5440,9 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v3", "coverageGaps": [], - "expectedRunCount": 3, + "expectedRunCount": 4, "turnTimeoutMs": 120000, "attemptTimeoutMs": 900000, "authentication": "human-session", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts index 4a9485379bb..d954a4ce23b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts @@ -1,11 +1,11 @@ import { expect, it } from "vitest"; -import { gradeApprovalQueue, acornNamingDirection, ordinaryApprovalEntries } from "./runtime-approval-queue.js"; +import { gradeApprovalQueue, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js"; function valid() { const payload = { questionSet: { questions: [{ id: "name", prompt: "What is the product name?" }] } }; return { scenarioId: "steering.approval-delivery", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], - runs: [{ id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }], - state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true, - questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "prior", payload }, + runs: [{ id: "asked", status: "succeeded" }, { id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }], + state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", questionRunId: "asked", questionRunYielded: true, workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true, + questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "asked", payload }, questionAfter: { id: "question", status: "answered", resolvedByUserId: "human" as string | null, resolvedByAgentId: null as string | null, payload, result: { answers: [{ questionId: "name", otherText: "Acorn" }] } }, responseRetainedAcrossOrdinaryMutations: true, responseUneditable: true, queryReply: "Use Acorn for the existing naming question; submit that bound answer through its supported response control.", actualOrdinaryEdit: true, actualOrdinaryReorder: true, ordinaryReorderRestricted: false, actualOrdinaryDiscard: true, humanAnswerSubmitted: true, canonicalAnswerAttributed: false, @@ -13,13 +13,12 @@ function valid() { } it("accepts real human response retained separately from mutated ordinary comments", () => expect(gradeApprovalQueue(valid()).passed).toBe(true)); it("accepts a permitted actual canonical agent response with native source attribution", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false, queryReply: "I answered Morgan's naming question with Acorn; the response is saved for normal continuation." }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); expect(gradeApprovalQueue(o).passed).toBe(true); }); -it("accepts natural correct-answer guidance", () => { const o = valid(); o.state.queryReply = "The product name is Acorn. Use the question's response card so this decision stays attached to the naming request."; expect(gradeApprovalQueue(o).passed).toBe(true); }); it("rejects a fabricated question without actual prior native provenance", () => { const o = valid(); o.state.questionBefore.createdByAgentId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects an ordinary comment masquerading as bound response", () => { const o = valid(); o.state.responseQueue.entries[0]!.source.kind = "comment"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a canonical response recording a different name", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); o.state.questionAfter.result.answers[0]!.otherText = "Birch"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects wrong accepted answer", () => { const o = valid(); o.state.questionAfter.result.answers[0]!.otherText = "Other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects revoking the decision by discarding ordinary text", () => { const o = valid(); o.state.responseRetainedAcrossOrdinaryMutations = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects an editable interaction response", () => { const o = valid(); o.state.responseQueue.entries[0]!.canEdit = true; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it("rejects source claim of answering without an attributed actual answer", () => { const o = valid(); o.state.queryReply = "I have answered with Acorn."; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects unsupported actor answering for the human", () => { const o = valid(); o.state.questionAfter.resolvedByUserId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects ordinary message operations performed on a different queue", () => { const o = valid(); o.state.actualOrdinaryReorder = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects duplicate decision consumers", () => { const o = valid(); o.state.consumingRunCount = 2; expect(gradeApprovalQueue(o).passed).toBe(false); }); @@ -27,8 +26,6 @@ it("rejects native input lacking the bound question ID and answer", () => { cons it("rejects loss of earlier settled work", () => { const o = valid(); o.state.initialWorkRetained = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects missing actual response evidence", () => expect(gradeApprovalQueue(undefined).passed).toBe(false)); -it.each(["Use Birch, not Acorn, as the name", "Acorn is wrong; use Birch for the product name", "The requested name was Acorn; Use Birch instead to answer the question"])("rejects contradictory named direction without rescue by later human input", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it.each(["Use Acorn, not Birch, as the product name", "The answer is Acorn; submit it on the naming question", "Name it Acorn and retain the actual answer on the response card"])("accepts natural correct naming direction", reply => expect(acornNamingDirection(reply)).toBe(true)); it("restricts every mutating ID to ordinary entries in a mixed two-comment plus response queue", () => { const ordinary = { comment: { id: "ordinary" } }, response = { source: { kind: "interaction" }, comment: { id: "decision" } }; const ids = ordinaryApprovalEntries({ entries: [ordinary, response, { comment: { id: "other" } }] } as Parameters[0]).map(row => row.comment.id); @@ -36,4 +33,21 @@ it("restricts every mutating ID to ordinary entries in a mixed two-comment plus }); it("accepts the real mixed-queue UI reorder refusal while preserving immutable decision placement", () => { const o = valid(); o.state.actualOrdinaryReorder = false; o.state.ordinaryReorderRestricted = true; expect(gradeApprovalQueue(o).passed).toBe(true); }); -it("accepts explicit rejection of a different name while selecting the actual answer", () => expect(acornNamingDirection("Do not use Birch; the product name is Acorn.")).toBe(true)); + +// The canonical reply is read by a person, not matched by words: the verdict rests on the saved answer path. +it.each(["Use Acorn for the existing naming question; submit that bound answer through its supported response control.", "Open Morgan's naming card and send Acorn there so the decision stays attached to it.", "Got it."])("verdict does not depend on reply wording: %s", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(true); }); +it("rejects a question whose run never parked on it", () => { const o = valid(); o.state.questionRunYielded = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a question attributed to the later active turn instead of the turn that asked", () => { const o = valid(); o.state.questionBefore.sourceRunId = "prior"; o.state.questionRunId = "prior"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects an answer with no actual recording path", () => { const o = valid(); o.state.humanAnswerSubmitted = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a missing parked-question turn in the run ledger", () => { const o = valid(); o.runs.shift(); expect(gradeApprovalQueue(o).passed).toBe(false); }); + +// Shape recorded by the retained runner-opencode attempt (gha-42926276473): request_human_input parks the turn. +const parked = { id: "asked", status: "succeeded", resultJson: { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }], + artifacts: [{ ref: "interaction:question", kind: "issue_thread_interaction" }], continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } } }; +it("recognizes the product's parked pending-input turn", () => expect(pendingInputYield(parked, "question")).toBe(true)); +it.each([ + ["still running", { ...parked, status: "running" }], + ["completed instead of yielded", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, reportedWorkDisposition: "completed" } } }], + ["parked on another interaction", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, continuation: { kind: "response_wake", idempotencyKey: "interaction-response:other" } } } }], + ["no result", { id: "asked", status: "succeeded", resultJson: null }], +])("rejects a run that is not parked on the question (%s)", (_label, run) => expect(pendingInputYield(run, "question")).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts index a80a364f69d..0f72cceeed4 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts @@ -8,11 +8,17 @@ import { checks, type Entity } from "./internal-support.js"; import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; -export function acornNamingDirection(reply: string) { - if (/\b(?:not|instead of|rather than|avoid|reject)\s+(?:the name\s+)?Acorn\b|\bAcorn\s+(?:is|was|would be)\s+(?:the\s+)?(?:wrong|incorrect|invalid|rejected)\b/i.test(reply)) return false; - const selectedNames = [...reply.matchAll(/\b(?:[Uu]se|[Cc]hoose|[Nn]ame it|[Cc]all it|[Aa]nswer with)\s+["']?([A-Z][a-zA-Z0-9-]*)/g)].filter(match => !/\b(?:not|don't|avoid)\s*$/i.test(reply.slice(Math.max(0, (match.index ?? 0) - 12), match.index))).map(match => match[1]!); - if (selectedNames.some(name => !["Acorn", "The", "Submit", "That", "Its", "This"].includes(name))) return false; - return /\bAcorn\b/i.test(reply) && /question|answer|respond|decision|nam(?:e|ing)/i.test(reply); +type PendingInputRun = { id?: string; status?: string; resultJson?: { nativeResult?: { reportedWorkDisposition?: string; continuation?: { kind?: string; idempotencyKey?: string }; + evidence?: Array<{ ref?: string }>; artifacts?: Array<{ ref?: string }> } } | null }; +/** + * A native worker that saves a durable wake-based question parks its turn: the run + * succeeds with a yielded disposition whose continuation is the question's response + * wake. That saved yield, not a still-running process, is the product's pending-input state. + */ +export function pendingInputYield(run: PendingInputRun | null | undefined, questionId: string) { + const result = run?.resultJson?.nativeResult, ref = `interaction:${questionId}`; + return run?.status === "succeeded" && result?.reportedWorkDisposition === "yielded" && result.continuation?.kind === "response_wake" + && result.continuation.idempotencyKey === `interaction-response:${questionId}` && [...(result.evidence ?? []), ...(result.artifacts ?? [])].some(row => row.ref === ref); } export function ordinaryApprovalEntries(queue: Pick) { return queue.entries.filter(row => !row.source); } export function gradeApprovalQueue(value: unknown) { @@ -20,54 +26,71 @@ export function gradeApprovalQueue(value: unknown) { c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result(); const s = o.state, question = s.questionBefore as IssueThreadInteraction, answered = s.questionAfter as IssueThreadInteraction; c.add("genuine-prior-worker-question-and-saved-comments", !!question && question.kind === "ask_user_questions" && question.status === "pending" && question.issueId === (s.task as Entity).id - && question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.workerRunId && s.workerWasReallyWaiting === true + && question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.questionRunId && s.questionRunYielded === true + && typeof s.workerRunId === "string" && s.workerRunId !== s.questionRunId && s.workerWasReallyWaiting === true && s.ordinaryQueueOwned === true && s.ordinaryQueuedBeforeAnswer === true, - "Morgan's actual native turn asks the naming question and remains active while the real human's separate ordinary comments are pending."); + "Morgan's actual native turn saves the naming question and parks on it; a later real Morgan turn is still active while the human's separate ordinary comments wait."); const results = answered?.result as { answers?: Array<{ questionId?: string; otherText?: string }> } | null; c.add("actual-bound-human-answer-preserved", answered?.id === question.id && answered.status === "answered" && ((answered.resolvedByUserId === s.humanId && !answered.resolvedByAgentId && s.humanAnswerSubmitted === true) || (answered.resolvedByAgentId === o.leadId && s.canonicalAnswerAttributed === true)) && JSON.stringify(answered.payload) === JSON.stringify(question.payload) && results?.answers?.some(row => row.questionId === "name" && row.otherText === "Acorn") === true && s.responseRetainedAcrossOrdinaryMutations === true && s.responseUneditable === true, "Acorn remains the same immutable typed interaction response, distinct from the editable/deletable ordinary comment queue."); - c.add("human-guidance-and-real-ordinary-mutations", acornNamingDirection(String(s.queryReply)) - && (s.canonicalAnswerAttributed === true || !/I(?:'ve| have)? (?:already )?(?:answered|submitted|accepted)/i.test(String(s.queryReply))) - && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true && (s.humanAnswerSubmitted === true || s.canonicalAnswerAttributed === true), - "Canonical chat identifies the intended answer and actual human response control; editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); + c.add("bound-answer-path-and-real-ordinary-mutations", (s.humanAnswerSubmitted === true) !== (s.canonicalAnswerAttributed === true) + && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true, + "Exactly one actual path records Acorn on the bound question (the human's response control, or an attributed canonical response); editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); const response = s.responseQueue as IssueQueuedCommentQueue; c.add("response-own-delivery-semantics-and-once-only-admission", response?.entries.length === 1 && response.entries[0]?.source?.kind === "interaction" && response.entries[0].source.interactionId === question.id && response.entries[0].canEdit === false && response.entries[0].canDiscard === false && s.consumingRunCount === 1 && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true - && o.runs.length === 3 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, + && o.runs.length === 4 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, "After the original owner settles, exactly one eligible native continuation receives the bound answer and saves actual Acorn work without replaying or losing it."); return c.result(); } export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => { const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }), root = `/api/companies/${ctx.input.fixtures.company.id}`; const task = await ctx.input.api.post(`${root}/issues`, { title: "Website naming task", status: "todo", assigneeAgentId: prepared.worker.id, - description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run ${prepared.gate.command} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` }); - const waiting = await pollUntil({ label: "Real native naming question while worker waits in its own tool process", deadlineAt: Date.now() + 180_000, - load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await ctx.input.api.get(`/api/issues/${task.id}/interactions`) }), - accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.runs[0].status === "running" && v.ready === "waiting" - && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id, - reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "succeeded", "timed_out"].includes(row.status)) ? "The actual question fixture did not reach its running pending-input boundary" : undefined }); + description: `We need a product name before continuing the website. If this task has no naming question yet, create exactly one pending ask_user_questions interaction on it, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. If a human message wakes you while that naming question is still pending, run ${prepared.gate.command} once and wait for its actual output, then follow that output, leaving queued human answers for normal continuation. On a later normal continuation that delivers the submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` }); + const interactions = () => ctx.input.api.get(`/api/issues/${task.id}/interactions`); + // The product parks a native turn as soon as its durable wake-based question is + // saved, so the pending-input boundary is the saved yield, not a running process. + const asked = await pollUntil({ label: "Real native naming question parks Morgan's turn on pending input", deadlineAt: Date.now() + 180_000, + load: async () => { const runs = await ctx.allRuns(), questions = await interactions(); + const run = runs.length === 1 && runs[0]?.status === "succeeded" ? await ctx.input.api.get(`/api/heartbeat-runs/${runs[0].id}`) : null; + return { runs, questions, run }; }, + accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" + && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id && pendingInputYield(v.run, v.questions[0].id), + reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || (v.run && !(v.questions.length === 1 && pendingInputYield(v.run, v.questions[0]!.id))) + ? "The actual question fixture did not park its native turn on one pending naming question" : undefined }); + const question = asked.questions[0]!, questionRunId = asked.runs[0]!.id; const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(form).toBeVisible(); await form.getByRole("button", { name: /^Dismiss / }).click(); + // An ordinary human message while the question stays open is normal discussion; it wakes Morgan into a new turn. + await submitTaskReply(ctx.input.page, "Keep the hero section simple while the name is pending."); + const busy = await pollUntil({ label: "Real Morgan turn active in its own tool process while the naming question stays pending", deadlineAt: Date.now() + 180_000, + load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await interactions() }), + accept: v => v.runs.length === 2 && v.runs.some(row => row.id !== questionRunId && row.agentId === prepared.worker.id && row.status === "running") && v.ready === "waiting" + && v.questions.length === 1 && v.questions[0]?.id === question.id && v.questions[0].status === "pending", + reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || v.runs.some(row => row.id !== questionRunId && row.status === "succeeded") + || v.questions.length !== 1 || v.questions[0]?.status !== "pending" ? "The discussion turn did not stay active behind the pending naming question" : undefined }); + const workerRunId = busy.runs.find(row => row.id !== questionRunId)!.id; const messages = ["Preserve the original layout work.", "Use accessible labels."]; for (const body of messages) await submitTaskReply(ctx.input.page, body); const queue = await pollUntil({ label: "Actual ordinary human comments remain separate behind running Morgan", deadlineAt: Date.now() + 20_000, - load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), accept: q => q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) }); + load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), + accept: q => q.targetRunId === workerRunId && q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); if (queue.entries.some(row => row.comment.authorUserId !== session.user.id || !row.canEdit || !row.canDiscard)) throw new Error("Ordinary queue is not actually owned by the signed-in human"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - return { worker: prepared.worker, gate: prepared.gate, task, workerRunId: waiting.runs[0]!.id, workerWasReallyWaiting: true, questionBefore: waiting.questions[0]!, ordinaryQueueBefore: queue, + return { worker: prepared.worker, gate: prepared.gate, task, questionRunId, questionRunYielded: true, workerRunId, workerWasReallyWaiting: true, questionBefore: question, ordinaryQueueBefore: queue, humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true }; }, gradeApprovalQueue, { setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, + authentication: "human-session", expectedRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v3", sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); state.queryReply = (await observeRuntime(ctx, state)).reply; const task = state.task as Entity, question = state.questionBefore as IssueThreadInteraction, source = ctx.input.page.url(), endpoint = `/api/issues/${task.id}/queued-comments`; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); @@ -126,12 +149,12 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled(); await ctx.input.capture("actual-immutable-answer-queue", "Bound Acorn answer remains distinct from edited, reordered and removed ordinary comments", "runtime-approval-queue.png"); state.initialMarker = `Earlier website work ${ctx.input.nonce}`; - await writeFile((state.gate as { gate: string }).gate, `The current pending-input turn is settling. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); + await writeFile((state.gate as { gate: string }).gate, `This turn is settling while the naming question waits. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", - reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); - const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; + accept: v => v.runs.length === 4 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", + reject: v => v.runs.length > 4 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); + const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId && row.id !== state.questionRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; const full = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { interactionId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`); state.responseNativeInputBound = full?.contextSnapshot?.interactionId === question.id && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(question.id) === true && full.runnerProfileJson.nativeExecutionInput.task.prompt.includes("Acorn"); state.queueEmptyAfterDelivery = done.queue.entries.length === 0; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts index c1a76bf9980..c4299abdd8f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts @@ -7,11 +7,13 @@ const prepare = vi.hoisted(() => vi.fn()); const poll = vi.hoisted(() => vi.fn()); vi.mock("./runtime-busy-worker.js", async original => ({ ...await original(), prepareRuntimeWorkerGate: prepare })); vi.mock("../api.js", async original => ({ ...await original(), pollUntil: poll })); -import { taskBoundRuntimeGate } from "./runtime-busy-worker.js"; +import { readRuntimeTaskGate, taskBoundRuntimeGate } from "./runtime-busy-worker.js"; import { runtimeApprovalQueueCases } from "./runtime-approval-queue.js"; const roots: string[] = []; -afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function fixture() { +afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); poll.mockReset(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const parkedResult = { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }], + continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } }; +async function fixture(run: { status: string; resultJson: unknown } = { status: "succeeded", resultJson: parkedResult }) { const root = await mkdtemp(path.join(tmpdir(), "approval-owned-task-")); roots.push(root); vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root); const cwd = path.join(root, "actual-isolated-task"); await mkdir(cwd); const gate = taskBoundRuntimeGate("approval-owned"); @@ -20,26 +22,39 @@ async function fixture() { const workspace = { id: "workspace", companyId: "company", cwd }; const post = vi.fn(async (_url: string, _body: unknown) => task); prepare.mockImplementation(async (_ctx, options) => { expect(options).toEqual({ taskBound: true }); return { gate, worker: { id: "morgan" } }; }); - const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get: async (url: string) => url.endsWith("/workspace") ? { workspace } : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] : task } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: "running" }] } as unknown as ChatCatalogContext; + const get = async (url: string) => url.endsWith("/workspace") ? { workspace } + : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] + : url === "/api/heartbeat-runs/run" ? { id: "run", ...run } : task; + const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: run.status }] } as unknown as ChatCatalogContext; return { ctx, root, cwd, task, workspace, gate, post }; } -it("approval setup consumes actual assigned-workspace readiness using a cwd-local bounded command", async () => { +it("approval setup gates the question on the product's parked pending-input turn and scripts a cwd-local bounded wait", async () => { const f = await fixture(); - poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); throw new Error("verified active boundary"); }); - await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified active boundary"); + poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); expect(options.reject(value)).toBeUndefined(); throw new Error("verified pending-input boundary"); }); + await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified pending-input boundary"); expect(f.post.mock.calls[0]![1]).toMatchObject({ assigneeAgentId: "morgan", status: "todo" }); const body = f.post.mock.calls[0]![1] as { description: string }; expect(body.description).toContain(`run ${f.gate.command} once`); expect(body.description).not.toContain(f.root); +}); +it.each([ + ["a question turn that is still running", { status: "running", resultJson: null }, false], + ["a turn that completed without parking on the question", { status: "succeeded", resultJson: { nativeResult: { ...parkedResult.nativeResult, reportedWorkDisposition: "completed" } } }, true], +])("approval setup does not accept %s", async (_label, run, rejected) => { + const f = await fixture(run); + poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(false); expect(Boolean(options.reject(value))).toBe(rejected); throw new Error("checked"); }); + await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("checked"); +}); +it("the active-turn gate binds the actual assigned task workspace", async () => { + const f = await fixture(); + await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).resolves.toBe("waiting"); expect(f.gate.boundTaskId).toBe("task"); expect(f.gate.boundWorkspaceId).toBe("workspace"); expect(f.gate.ready).toBe(path.join(await realpath(f.cwd), `${f.gate.gateName}.waiting`)); }); -it.each(["company", "assignee", "workspace"])("approval setup rejects foreign %s readiness before accepting the question boundary", async invalid => { +it.each(["company", "assignee", "workspace"])("the active-turn gate rejects foreign %s readiness", async invalid => { const f = await fixture(); if (invalid === "company") f.workspace.companyId = "foreign"; if (invalid === "assignee") f.task.assigneeAgentId = "other"; if (invalid === "workspace") f.task.executionWorkspaceId = "other"; - const accept = vi.fn(); poll.mockImplementationOnce(async options => { await options.load(); accept(); }); - await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow(/actual company, assignee and workspace/); - expect(accept).not.toHaveBeenCalled(); + await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).rejects.toThrow(/actual company, assignee and workspace/); });