diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..b49ec36fcc9 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5440,9 +5440,9 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v3", "coverageGaps": [], - "expectedRunCount": 3, + "expectedRunCount": 4, "turnTimeoutMs": 120000, "attemptTimeoutMs": 900000, "authentication": "human-session", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts index 4a9485379bb..d954a4ce23b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts @@ -1,11 +1,11 @@ import { expect, it } from "vitest"; -import { gradeApprovalQueue, acornNamingDirection, ordinaryApprovalEntries } from "./runtime-approval-queue.js"; +import { gradeApprovalQueue, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js"; function valid() { const payload = { questionSet: { questions: [{ id: "name", prompt: "What is the product name?" }] } }; return { scenarioId: "steering.approval-delivery", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], - runs: [{ id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }], - state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true, - questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "prior", payload }, + runs: [{ id: "asked", status: "succeeded" }, { id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }], + state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", questionRunId: "asked", questionRunYielded: true, workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true, + questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "asked", payload }, questionAfter: { id: "question", status: "answered", resolvedByUserId: "human" as string | null, resolvedByAgentId: null as string | null, payload, result: { answers: [{ questionId: "name", otherText: "Acorn" }] } }, responseRetainedAcrossOrdinaryMutations: true, responseUneditable: true, queryReply: "Use Acorn for the existing naming question; submit that bound answer through its supported response control.", actualOrdinaryEdit: true, actualOrdinaryReorder: true, ordinaryReorderRestricted: false, actualOrdinaryDiscard: true, humanAnswerSubmitted: true, canonicalAnswerAttributed: false, @@ -13,13 +13,12 @@ function valid() { } it("accepts real human response retained separately from mutated ordinary comments", () => expect(gradeApprovalQueue(valid()).passed).toBe(true)); it("accepts a permitted actual canonical agent response with native source attribution", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false, queryReply: "I answered Morgan's naming question with Acorn; the response is saved for normal continuation." }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); expect(gradeApprovalQueue(o).passed).toBe(true); }); -it("accepts natural correct-answer guidance", () => { const o = valid(); o.state.queryReply = "The product name is Acorn. Use the question's response card so this decision stays attached to the naming request."; expect(gradeApprovalQueue(o).passed).toBe(true); }); it("rejects a fabricated question without actual prior native provenance", () => { const o = valid(); o.state.questionBefore.createdByAgentId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects an ordinary comment masquerading as bound response", () => { const o = valid(); o.state.responseQueue.entries[0]!.source.kind = "comment"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a canonical response recording a different name", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); o.state.questionAfter.result.answers[0]!.otherText = "Birch"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects wrong accepted answer", () => { const o = valid(); o.state.questionAfter.result.answers[0]!.otherText = "Other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects revoking the decision by discarding ordinary text", () => { const o = valid(); o.state.responseRetainedAcrossOrdinaryMutations = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects an editable interaction response", () => { const o = valid(); o.state.responseQueue.entries[0]!.canEdit = true; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it("rejects source claim of answering without an attributed actual answer", () => { const o = valid(); o.state.queryReply = "I have answered with Acorn."; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects unsupported actor answering for the human", () => { const o = valid(); o.state.questionAfter.resolvedByUserId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects ordinary message operations performed on a different queue", () => { const o = valid(); o.state.actualOrdinaryReorder = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects duplicate decision consumers", () => { const o = valid(); o.state.consumingRunCount = 2; expect(gradeApprovalQueue(o).passed).toBe(false); }); @@ -27,8 +26,6 @@ it("rejects native input lacking the bound question ID and answer", () => { cons it("rejects loss of earlier settled work", () => { const o = valid(); o.state.initialWorkRetained = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects missing actual response evidence", () => expect(gradeApprovalQueue(undefined).passed).toBe(false)); -it.each(["Use Birch, not Acorn, as the name", "Acorn is wrong; use Birch for the product name", "The requested name was Acorn; Use Birch instead to answer the question"])("rejects contradictory named direction without rescue by later human input", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it.each(["Use Acorn, not Birch, as the product name", "The answer is Acorn; submit it on the naming question", "Name it Acorn and retain the actual answer on the response card"])("accepts natural correct naming direction", reply => expect(acornNamingDirection(reply)).toBe(true)); it("restricts every mutating ID to ordinary entries in a mixed two-comment plus response queue", () => { const ordinary = { comment: { id: "ordinary" } }, response = { source: { kind: "interaction" }, comment: { id: "decision" } }; const ids = ordinaryApprovalEntries({ entries: [ordinary, response, { comment: { id: "other" } }] } as Parameters[0]).map(row => row.comment.id); @@ -36,4 +33,21 @@ it("restricts every mutating ID to ordinary entries in a mixed two-comment plus }); it("accepts the real mixed-queue UI reorder refusal while preserving immutable decision placement", () => { const o = valid(); o.state.actualOrdinaryReorder = false; o.state.ordinaryReorderRestricted = true; expect(gradeApprovalQueue(o).passed).toBe(true); }); -it("accepts explicit rejection of a different name while selecting the actual answer", () => expect(acornNamingDirection("Do not use Birch; the product name is Acorn.")).toBe(true)); + +// The canonical reply is read by a person, not matched by words: the verdict rests on the saved answer path. +it.each(["Use Acorn for the existing naming question; submit that bound answer through its supported response control.", "Open Morgan's naming card and send Acorn there so the decision stays attached to it.", "Got it."])("verdict does not depend on reply wording: %s", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(true); }); +it("rejects a question whose run never parked on it", () => { const o = valid(); o.state.questionRunYielded = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a question attributed to the later active turn instead of the turn that asked", () => { const o = valid(); o.state.questionBefore.sourceRunId = "prior"; o.state.questionRunId = "prior"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects an answer with no actual recording path", () => { const o = valid(); o.state.humanAnswerSubmitted = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a missing parked-question turn in the run ledger", () => { const o = valid(); o.runs.shift(); expect(gradeApprovalQueue(o).passed).toBe(false); }); + +// Shape recorded by the retained runner-opencode attempt (gha-42926276473): request_human_input parks the turn. +const parked = { id: "asked", status: "succeeded", resultJson: { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }], + artifacts: [{ ref: "interaction:question", kind: "issue_thread_interaction" }], continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } } }; +it("recognizes the product's parked pending-input turn", () => expect(pendingInputYield(parked, "question")).toBe(true)); +it.each([ + ["still running", { ...parked, status: "running" }], + ["completed instead of yielded", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, reportedWorkDisposition: "completed" } } }], + ["parked on another interaction", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, continuation: { kind: "response_wake", idempotencyKey: "interaction-response:other" } } } }], + ["no result", { id: "asked", status: "succeeded", resultJson: null }], +])("rejects a run that is not parked on the question (%s)", (_label, run) => expect(pendingInputYield(run, "question")).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts index a80a364f69d..0f72cceeed4 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts @@ -8,11 +8,17 @@ import { checks, type Entity } from "./internal-support.js"; import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; -export function acornNamingDirection(reply: string) { - if (/\b(?:not|instead of|rather than|avoid|reject)\s+(?:the name\s+)?Acorn\b|\bAcorn\s+(?:is|was|would be)\s+(?:the\s+)?(?:wrong|incorrect|invalid|rejected)\b/i.test(reply)) return false; - const selectedNames = [...reply.matchAll(/\b(?:[Uu]se|[Cc]hoose|[Nn]ame it|[Cc]all it|[Aa]nswer with)\s+["']?([A-Z][a-zA-Z0-9-]*)/g)].filter(match => !/\b(?:not|don't|avoid)\s*$/i.test(reply.slice(Math.max(0, (match.index ?? 0) - 12), match.index))).map(match => match[1]!); - if (selectedNames.some(name => !["Acorn", "The", "Submit", "That", "Its", "This"].includes(name))) return false; - return /\bAcorn\b/i.test(reply) && /question|answer|respond|decision|nam(?:e|ing)/i.test(reply); +type PendingInputRun = { id?: string; status?: string; resultJson?: { nativeResult?: { reportedWorkDisposition?: string; continuation?: { kind?: string; idempotencyKey?: string }; + evidence?: Array<{ ref?: string }>; artifacts?: Array<{ ref?: string }> } } | null }; +/** + * A native worker that saves a durable wake-based question parks its turn: the run + * succeeds with a yielded disposition whose continuation is the question's response + * wake. That saved yield, not a still-running process, is the product's pending-input state. + */ +export function pendingInputYield(run: PendingInputRun | null | undefined, questionId: string) { + const result = run?.resultJson?.nativeResult, ref = `interaction:${questionId}`; + return run?.status === "succeeded" && result?.reportedWorkDisposition === "yielded" && result.continuation?.kind === "response_wake" + && result.continuation.idempotencyKey === `interaction-response:${questionId}` && [...(result.evidence ?? []), ...(result.artifacts ?? [])].some(row => row.ref === ref); } export function ordinaryApprovalEntries(queue: Pick) { return queue.entries.filter(row => !row.source); } export function gradeApprovalQueue(value: unknown) { @@ -20,54 +26,71 @@ export function gradeApprovalQueue(value: unknown) { c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result(); const s = o.state, question = s.questionBefore as IssueThreadInteraction, answered = s.questionAfter as IssueThreadInteraction; c.add("genuine-prior-worker-question-and-saved-comments", !!question && question.kind === "ask_user_questions" && question.status === "pending" && question.issueId === (s.task as Entity).id - && question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.workerRunId && s.workerWasReallyWaiting === true + && question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.questionRunId && s.questionRunYielded === true + && typeof s.workerRunId === "string" && s.workerRunId !== s.questionRunId && s.workerWasReallyWaiting === true && s.ordinaryQueueOwned === true && s.ordinaryQueuedBeforeAnswer === true, - "Morgan's actual native turn asks the naming question and remains active while the real human's separate ordinary comments are pending."); + "Morgan's actual native turn saves the naming question and parks on it; a later real Morgan turn is still active while the human's separate ordinary comments wait."); const results = answered?.result as { answers?: Array<{ questionId?: string; otherText?: string }> } | null; c.add("actual-bound-human-answer-preserved", answered?.id === question.id && answered.status === "answered" && ((answered.resolvedByUserId === s.humanId && !answered.resolvedByAgentId && s.humanAnswerSubmitted === true) || (answered.resolvedByAgentId === o.leadId && s.canonicalAnswerAttributed === true)) && JSON.stringify(answered.payload) === JSON.stringify(question.payload) && results?.answers?.some(row => row.questionId === "name" && row.otherText === "Acorn") === true && s.responseRetainedAcrossOrdinaryMutations === true && s.responseUneditable === true, "Acorn remains the same immutable typed interaction response, distinct from the editable/deletable ordinary comment queue."); - c.add("human-guidance-and-real-ordinary-mutations", acornNamingDirection(String(s.queryReply)) - && (s.canonicalAnswerAttributed === true || !/I(?:'ve| have)? (?:already )?(?:answered|submitted|accepted)/i.test(String(s.queryReply))) - && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true && (s.humanAnswerSubmitted === true || s.canonicalAnswerAttributed === true), - "Canonical chat identifies the intended answer and actual human response control; editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); + c.add("bound-answer-path-and-real-ordinary-mutations", (s.humanAnswerSubmitted === true) !== (s.canonicalAnswerAttributed === true) + && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true, + "Exactly one actual path records Acorn on the bound question (the human's response control, or an attributed canonical response); editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); const response = s.responseQueue as IssueQueuedCommentQueue; c.add("response-own-delivery-semantics-and-once-only-admission", response?.entries.length === 1 && response.entries[0]?.source?.kind === "interaction" && response.entries[0].source.interactionId === question.id && response.entries[0].canEdit === false && response.entries[0].canDiscard === false && s.consumingRunCount === 1 && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true - && o.runs.length === 3 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, + && o.runs.length === 4 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, "After the original owner settles, exactly one eligible native continuation receives the bound answer and saves actual Acorn work without replaying or losing it."); return c.result(); } export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => { const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }), root = `/api/companies/${ctx.input.fixtures.company.id}`; const task = await ctx.input.api.post(`${root}/issues`, { title: "Website naming task", status: "todo", assigneeAgentId: prepared.worker.id, - description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run ${prepared.gate.command} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` }); - const waiting = await pollUntil({ label: "Real native naming question while worker waits in its own tool process", deadlineAt: Date.now() + 180_000, - load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await ctx.input.api.get(`/api/issues/${task.id}/interactions`) }), - accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.runs[0].status === "running" && v.ready === "waiting" - && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id, - reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "succeeded", "timed_out"].includes(row.status)) ? "The actual question fixture did not reach its running pending-input boundary" : undefined }); + description: `We need a product name before continuing the website. If this task has no naming question yet, create exactly one pending ask_user_questions interaction on it, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. If a human message wakes you while that naming question is still pending, run ${prepared.gate.command} once and wait for its actual output, then follow that output, leaving queued human answers for normal continuation. On a later normal continuation that delivers the submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` }); + const interactions = () => ctx.input.api.get(`/api/issues/${task.id}/interactions`); + // The product parks a native turn as soon as its durable wake-based question is + // saved, so the pending-input boundary is the saved yield, not a running process. + const asked = await pollUntil({ label: "Real native naming question parks Morgan's turn on pending input", deadlineAt: Date.now() + 180_000, + load: async () => { const runs = await ctx.allRuns(), questions = await interactions(); + const run = runs.length === 1 && runs[0]?.status === "succeeded" ? await ctx.input.api.get(`/api/heartbeat-runs/${runs[0].id}`) : null; + return { runs, questions, run }; }, + accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" + && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id && pendingInputYield(v.run, v.questions[0].id), + reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || (v.run && !(v.questions.length === 1 && pendingInputYield(v.run, v.questions[0]!.id))) + ? "The actual question fixture did not park its native turn on one pending naming question" : undefined }); + const question = asked.questions[0]!, questionRunId = asked.runs[0]!.id; const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(form).toBeVisible(); await form.getByRole("button", { name: /^Dismiss / }).click(); + // An ordinary human message while the question stays open is normal discussion; it wakes Morgan into a new turn. + await submitTaskReply(ctx.input.page, "Keep the hero section simple while the name is pending."); + const busy = await pollUntil({ label: "Real Morgan turn active in its own tool process while the naming question stays pending", deadlineAt: Date.now() + 180_000, + load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await interactions() }), + accept: v => v.runs.length === 2 && v.runs.some(row => row.id !== questionRunId && row.agentId === prepared.worker.id && row.status === "running") && v.ready === "waiting" + && v.questions.length === 1 && v.questions[0]?.id === question.id && v.questions[0].status === "pending", + reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || v.runs.some(row => row.id !== questionRunId && row.status === "succeeded") + || v.questions.length !== 1 || v.questions[0]?.status !== "pending" ? "The discussion turn did not stay active behind the pending naming question" : undefined }); + const workerRunId = busy.runs.find(row => row.id !== questionRunId)!.id; const messages = ["Preserve the original layout work.", "Use accessible labels."]; for (const body of messages) await submitTaskReply(ctx.input.page, body); const queue = await pollUntil({ label: "Actual ordinary human comments remain separate behind running Morgan", deadlineAt: Date.now() + 20_000, - load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), accept: q => q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) }); + load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), + accept: q => q.targetRunId === workerRunId && q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); if (queue.entries.some(row => row.comment.authorUserId !== session.user.id || !row.canEdit || !row.canDiscard)) throw new Error("Ordinary queue is not actually owned by the signed-in human"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - return { worker: prepared.worker, gate: prepared.gate, task, workerRunId: waiting.runs[0]!.id, workerWasReallyWaiting: true, questionBefore: waiting.questions[0]!, ordinaryQueueBefore: queue, + return { worker: prepared.worker, gate: prepared.gate, task, questionRunId, questionRunYielded: true, workerRunId, workerWasReallyWaiting: true, questionBefore: question, ordinaryQueueBefore: queue, humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true }; }, gradeApprovalQueue, { setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000, + authentication: "human-session", expectedRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v3", sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); state.queryReply = (await observeRuntime(ctx, state)).reply; const task = state.task as Entity, question = state.questionBefore as IssueThreadInteraction, source = ctx.input.page.url(), endpoint = `/api/issues/${task.id}/queued-comments`; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); @@ -126,12 +149,12 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled(); await ctx.input.capture("actual-immutable-answer-queue", "Bound Acorn answer remains distinct from edited, reordered and removed ordinary comments", "runtime-approval-queue.png"); state.initialMarker = `Earlier website work ${ctx.input.nonce}`; - await writeFile((state.gate as { gate: string }).gate, `The current pending-input turn is settling. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); + await writeFile((state.gate as { gate: string }).gate, `This turn is settling while the naming question waits. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", - reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); - const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; + accept: v => v.runs.length === 4 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", + reject: v => v.runs.length > 4 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); + const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId && row.id !== state.questionRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; const full = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { interactionId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`); state.responseNativeInputBound = full?.contextSnapshot?.interactionId === question.id && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(question.id) === true && full.runnerProfileJson.nativeExecutionInput.task.prompt.includes("Acorn"); state.queueEmptyAfterDelivery = done.queue.entries.length === 0; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts index c1a76bf9980..c4299abdd8f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts @@ -7,11 +7,13 @@ const prepare = vi.hoisted(() => vi.fn()); const poll = vi.hoisted(() => vi.fn()); vi.mock("./runtime-busy-worker.js", async original => ({ ...await original(), prepareRuntimeWorkerGate: prepare })); vi.mock("../api.js", async original => ({ ...await original(), pollUntil: poll })); -import { taskBoundRuntimeGate } from "./runtime-busy-worker.js"; +import { readRuntimeTaskGate, taskBoundRuntimeGate } from "./runtime-busy-worker.js"; import { runtimeApprovalQueueCases } from "./runtime-approval-queue.js"; const roots: string[] = []; -afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function fixture() { +afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); poll.mockReset(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const parkedResult = { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }], + continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } }; +async function fixture(run: { status: string; resultJson: unknown } = { status: "succeeded", resultJson: parkedResult }) { const root = await mkdtemp(path.join(tmpdir(), "approval-owned-task-")); roots.push(root); vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root); const cwd = path.join(root, "actual-isolated-task"); await mkdir(cwd); const gate = taskBoundRuntimeGate("approval-owned"); @@ -20,26 +22,39 @@ async function fixture() { const workspace = { id: "workspace", companyId: "company", cwd }; const post = vi.fn(async (_url: string, _body: unknown) => task); prepare.mockImplementation(async (_ctx, options) => { expect(options).toEqual({ taskBound: true }); return { gate, worker: { id: "morgan" } }; }); - const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get: async (url: string) => url.endsWith("/workspace") ? { workspace } : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] : task } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: "running" }] } as unknown as ChatCatalogContext; + const get = async (url: string) => url.endsWith("/workspace") ? { workspace } + : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] + : url === "/api/heartbeat-runs/run" ? { id: "run", ...run } : task; + const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: run.status }] } as unknown as ChatCatalogContext; return { ctx, root, cwd, task, workspace, gate, post }; } -it("approval setup consumes actual assigned-workspace readiness using a cwd-local bounded command", async () => { +it("approval setup gates the question on the product's parked pending-input turn and scripts a cwd-local bounded wait", async () => { const f = await fixture(); - poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); throw new Error("verified active boundary"); }); - await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified active boundary"); + poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); expect(options.reject(value)).toBeUndefined(); throw new Error("verified pending-input boundary"); }); + await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified pending-input boundary"); expect(f.post.mock.calls[0]![1]).toMatchObject({ assigneeAgentId: "morgan", status: "todo" }); const body = f.post.mock.calls[0]![1] as { description: string }; expect(body.description).toContain(`run ${f.gate.command} once`); expect(body.description).not.toContain(f.root); +}); +it.each([ + ["a question turn that is still running", { status: "running", resultJson: null }, false], + ["a turn that completed without parking on the question", { status: "succeeded", resultJson: { nativeResult: { ...parkedResult.nativeResult, reportedWorkDisposition: "completed" } } }, true], +])("approval setup does not accept %s", async (_label, run, rejected) => { + const f = await fixture(run); + poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(false); expect(Boolean(options.reject(value))).toBe(rejected); throw new Error("checked"); }); + await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("checked"); +}); +it("the active-turn gate binds the actual assigned task workspace", async () => { + const f = await fixture(); + await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).resolves.toBe("waiting"); expect(f.gate.boundTaskId).toBe("task"); expect(f.gate.boundWorkspaceId).toBe("workspace"); expect(f.gate.ready).toBe(path.join(await realpath(f.cwd), `${f.gate.gateName}.waiting`)); }); -it.each(["company", "assignee", "workspace"])("approval setup rejects foreign %s readiness before accepting the question boundary", async invalid => { +it.each(["company", "assignee", "workspace"])("the active-turn gate rejects foreign %s readiness", async invalid => { const f = await fixture(); if (invalid === "company") f.workspace.companyId = "foreign"; if (invalid === "assignee") f.task.assigneeAgentId = "other"; if (invalid === "workspace") f.task.executionWorkspaceId = "other"; - const accept = vi.fn(); poll.mockImplementationOnce(async options => { await options.load(); accept(); }); - await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow(/actual company, assignee and workspace/); - expect(accept).not.toHaveBeenCalled(); + await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).rejects.toThrow(/actual company, assignee and workspace/); });