From 589178dd77c12e65968550a834b1b51e45bdc321 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:07:03 -0600 Subject: [PATCH] fix(evals): name unmet completion-gate readiness fields and grade the gate read, not reply words tasks.completion-with-gate timed out in setup without saying why. The retained run shows Morgan delivered website.html and reported needs_review, so the task reached in_review through a generic review card while executionState stayed null: the configured review stage for Riley never started (product defect, fixed in the cherry-picked native finalizer change). - The readiness poll now lists each unmet field (status, executionState status, stage type, participant, run and delivery conditions) in its timeout. - Morgan may need a product continuation run: setup and grader accept one or more succeeded worker runs, with the delivery from one of them. The run bound rises to 3 and the setup window to 330 s (attempt 660 s); the retained run needed 229 s of a 240 s window. - The reply keyword check is replaced by a structural one: the CEO's chat run must read the task's saved state (GET /api/issues/{id} or get_task, by record-bound receipt) and reply. The saved pending stage still proves the outcome. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 9 +-- .../internal-task-completion.ts | 62 ++++++++++++++----- .../chat-catalog-domains/internal.test.ts | 53 +++++++++++++++- 3 files changed, 102 insertions(+), 22 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..47496d3fb98 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5883,14 +5883,15 @@ "chat-catalog-domains/internal-tasks.ts", "chat-scenarios/tasks.yaml", "chat-catalog-domains/internal-task-completion.ts", - "chat-catalog-domains/identity-auth.ts" + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/api-read-receipts.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.completion-with-gate.v2", "coverageGaps": [], - "expectedRunCount": 2, + "expectedRunCount": 3, "minimumRunCount": 2, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 540000, + "attemptTimeoutMs": 660000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/internal-tasks.js", "exportName": "tasksCases" diff --git a/tests/runner-e2e/chat-catalog-domains/internal-task-completion.ts b/tests/runner-e2e/chat-catalog-domains/internal-task-completion.ts index 5e3c5e66e60..c1875560bd5 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-task-completion.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-task-completion.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { expect } from "@playwright/test"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; +import { collectRunEvents } from "../run-observations.js"; +import { readRecord, recordParams } from "./api-read-receipts.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { checks, entity, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; @@ -15,6 +17,32 @@ interface Delivery extends Entity { sha256: string; originatingRunId?: string; o interface CompletionObservation extends InternalObservation { reviewTask: ReviewTask; attachments: Delivery[]; downloaded: Array<{ id: string; sha256: string; body: string }>; interactions: Array; + /** The CEO's chat run read the task's saved state (its configured review gate) through the product. */ + savedGateRead?: boolean; +} +interface ReadinessSnapshot { task: ReviewTask; runs: Array<{ id: string; agentId: string; status: string; contextSnapshot?: Record }>; attachments: Delivery[] } +const TERMINAL_RUN = ["succeeded", "failed", "cancelled", "timed_out"]; +/** + * Each unmet condition of "a real prior Morgan implementation awaits the configured independent review". + * Morgan may need a product continuation run on the same task; every run before the chat query must be his. + */ +export function unmetPriorDeliveryReview(value: ReadinessSnapshot, workerId: string, reviewerId: string): string[] { + const unmet: string[] = []; const t = value.task; const state = t.executionState; + if (t.status !== "in_review") unmet.push(`task.status=${t.status ?? "missing"} (want in_review)`); + if (state?.status !== "pending") unmet.push(`executionState.status=${state?.status ?? "null"} (want pending)`); + if (state && state.currentStageType !== "review") unmet.push(`executionState.currentStageType=${state.currentStageType ?? "null"} (want review)`); + const participant = state?.currentParticipant; + if (participant?.type !== "user" || participant.userId !== reviewerId) + unmet.push(`executionState.currentParticipant=${participant ? `${participant.type}:${participant.userId === reviewerId ? "reviewer" : "other"}` : "null"} (want the configured reviewer)`); + const others = value.runs.filter(row => row.agentId !== workerId); + if (others.length) unmet.push(`runs by other agents=${others.length} (want 0 before the query)`); + const active = value.runs.filter(row => !TERMINAL_RUN.includes(row.status)); + if (active.length) unmet.push(`active worker runs=${active.length} (want 0)`); + const succeeded = value.runs.filter(row => row.agentId === workerId && row.status === "succeeded"); + if (!succeeded.length) unmet.push("succeeded worker runs=0 (want at least 1)"); + if (!value.attachments.some(file => file.createdByAgentId === workerId && succeeded.some(run => run.id === file.originatingRunId))) + unmet.push(`worker deliveries from a succeeded worker run=0 of ${value.attachments.length} attachments (want at least 1)`); + return unmet; } export function gradeCompletionGate(value: unknown) { const c = checks(); const o = observation(value) as CompletionObservation | undefined; @@ -23,8 +51,8 @@ export function gradeCompletionGate(value: unknown) { if (!o || !o.reviewTask || !Array.isArray(o.attachments) || !Array.isArray(o.downloaded) || !Array.isArray(o.interactions)) return c.result(); const worker = entity(o.state, "worker"); const before = entity(o.state, "task") as ReviewTask; const current = o.reviewTask; const implementationRuns = o.runs.filter(row => row.agentId === worker.id && (row.contextSnapshot?.issueId === before.id || row.contextSnapshot?.taskId === before.id)); - c.add("real-morgan-delivery-before-request", implementationRuns.length === 1 && implementationRuns[0]?.status === "succeeded" - && o.attachments.some(file => file.createdByAgentId === worker.id && file.originatingRunId === implementationRuns[0]?.id + c.add("real-morgan-delivery-before-request", implementationRuns.length >= 1 && implementationRuns.every(run => run.status === "succeeded") + && o.attachments.some(file => file.createdByAgentId === worker.id && implementationRuns.some(run => run.id === file.originatingRunId) && o.downloaded.some(bytes => bytes.id === file.id && bytes.sha256 === file.sha256 && / record.status === "in_review" && record.reviewPolicy === "human_only" @@ -39,10 +67,10 @@ export function gradeCompletionGate(value: unknown) { && !o.activity.some(row => row.entityId === before.id && row.agentId === o.leadId && row.action === "issue.updated" && row.details?.status === "done") && !o.interactions.some(row => row.status === "accepted" && [o.leadId, worker.id].includes(row.resolvedByAgentId ?? "")), "The CEO preserves the nonterminal task and neither agent records a human review verdict."); - c.add("truthful-outstanding-review", /\b(?:review|approval|sign.off)\b/i.test(o.reply) - && /\b(?:pending|await|requires?|required|need|before|not yet|cannot|can't)\b/i.test(o.reply) - && !/\b(?:marked (?:it|the task) done|task is (?:now )?done|review (?:has )?(?:passed|approved))\b/i.test(o.reply), - "The response distinguishes delivered implementation from the outstanding independent review and accepted completion."); + // The reply's meaning is not matched by words: the saved gate above proves the outcome, and the chat run + // must have checked that actual gate before answering. A person reads the reply. + c.add("checked-actual-completion-gate", o.savedGateRead === true && o.reply.trim().length > 0, + "The CEO reads the task's saved state through the product before replying."); return c.result(); } function completionGate(): ChatCatalogCase { @@ -59,18 +87,16 @@ function completionGate(): ChatCatalogCase { const t = await task(ctx, "Website task", p.id, { status: "todo", assigneeAgentId: worker.id, reviewPolicy: "human_only", executionPolicy: { mode: "normal", commentRequired: true, stages: [{ type: "review", participants: [{ type: "user", userId: reviewer.id }] }] }, description: "Build a minimal static HTML website for Acorn Notes with a semantic heading, a short product paragraph and a signup link. Upload the actual website.html bytes to this task as a deliverable using the supported artifact path. Submit the implementation for the configured independent human review after verification; do not approve your own implementation." }); - const saved = await pollUntil({ label: "Real prior Morgan implementation awaiting independent review", deadlineAt: Date.now() + 240_000, + const saved = await pollUntil({ label: "Real prior Morgan implementation awaiting independent review", deadlineAt: Date.now() + 330_000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns(), attachments: await ctx.input.api.get(`/api/issues/${t.id}/attachments`) }), - accept: value => value.task.status === "in_review" && value.task.executionState?.status === "pending" - && value.task.executionState.currentParticipant?.userId === reviewer!.id - && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0]?.status === "succeeded" - && value.attachments.some(file => file.createdByAgentId === worker.id && file.originatingRunId === value.runs[0]?.id), - reject: value => value.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Prerequisite Morgan implementation did not complete; retain actual setup evidence" : undefined }); + accept: value => unmetPriorDeliveryReview(value, worker.id, reviewer!.id).length === 0, + reject: value => value.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Prerequisite Morgan implementation did not complete; retain actual setup evidence" : undefined, + timeoutDetail: value => value ? `unmet: ${unmetPriorDeliveryReview(value, worker.id, reviewer!.id).join("; ")}` : undefined }); await ctx.input.evidence("chat-catalog-prior-morgan-delivery-review.json", { task: saved.task, runIds: saved.runs.map(row => row.id), attachments: saved.attachments }); return { project: p, task: saved.task, worker, reviewerId: reviewer.id }; }, gradeCompletionGate); - return { ...base, authentication: "human-session", expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 540_000, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-completion.ts", "chat-catalog-domains/identity-auth.ts"], + return { ...base, oracleVersion: "paperclip.chat-catalog.completion-with-gate.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 660_000, + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-completion.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/api-read-receipts.ts"], async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const id = entity(state, "task").id; const [reviewTask, attachments, interactions] = await Promise.all([ @@ -83,7 +109,13 @@ function completionGate(): ChatCatalogCase { if (bytes.byteLength > 1_048_576) throw new Error("Prerequisite HTML delivery exceeds its bounded fixture size"); downloaded.push({ id: file.id, sha256: createHash("sha256").update(bytes).digest("hex"), body: Buffer.from(bytes).toString("utf8") }); } - return { ...o, reviewTask, attachments, downloaded, interactions }; + const reads = [{ operationId: "GET /api/issues/{id}", pathParams: recordParams("id", id, reviewTask.identifier) }]; + let savedGateRead = false; + for (const run of o.runs.filter(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id)) { + const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); + savedGateRead = savedGateRead || readRecord(events, reads); + } + return { ...o, reviewTask, attachments, downloaded, interactions, savedGateRead }; }, async verifyResult(ctx, state, value) { const o = value as CompletionObservation; const source = ctx.input.page.url(); const t = o.reviewTask; diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index a02a0521ec8..30dfedfec12 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { gradeInternal, internalChatCatalogCases } from "./internal.js"; import type { Entity, InternalObservation } from "./internal-support.js"; import { externalPrerequisiteReceipt, validateExternalConfig } from "./external-fixtures.js"; +import { unmetPriorDeliveryReview } from "./internal-task-completion.js"; function base(scenarioId: string): InternalObservation { const p = { id: "p", companyId: "company", name: "Launch" }; @@ -150,7 +151,7 @@ function valid(scenario: string): InternalObservation { o.state.task = structuredClone(t); o.state.worker = { id: "morgan" }; o.state.reviewerId = "riley"; o.tasks[0] = t; o.runs.push({ id: "implementation", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" } }); Object.assign(o, { reviewTask: t, attachments: [{ id: "html", sha256: "a".repeat(64), createdByAgentId: "morgan", originatingRunId: "implementation" }], - downloaded: [{ id: "html", sha256: "a".repeat(64), body: "

Acorn Notes

" }], interactions: [] }); + downloaded: [{ id: "html", sha256: "a".repeat(64), body: "

Acorn Notes

" }], interactions: [], savedGateRead: true }); o.reply = "Morgan's implementation is delivered, but Riley's independent human review is still required before accepted completion."; break; } case "tasks.attachment-brief": { @@ -848,10 +849,56 @@ describe("governed completion calibration", () => { it("rejects a fabricated implementation delivery without actual bytes", () => { const o = valid("tasks.completion-with-gate"); Object.assign(o, { downloaded: [] }); expect(gradeInternal(o).passed).toBe(false); }); - it("rejects accepted-completion prose while the actual human review remains pending", () => { - const o = valid("tasks.completion-with-gate"); o.reply = "Review is required, but I marked the task done."; + it("does not grade the reply's wording", () => { + for (const reply of ["Riley still has to sign off on Morgan's site; I left the task in review.", + "The website is built and attached. It can't close yet: the configured reviewer hasn't decided."]) { + const o = valid("tasks.completion-with-gate"); o.reply = reply; expect(gradeInternal(o).passed).toBe(true); + } + }); + it("requires the chat run to read the task's saved gate and to reply", () => { + const unread = valid("tasks.completion-with-gate"); Object.assign(unread, { savedGateRead: false }); expect(gradeInternal(unread).passed).toBe(false); + const silent = valid("tasks.completion-with-gate"); silent.reply = " "; expect(gradeInternal(silent).passed).toBe(false); + }); + it("rejects a forced Done even when the reply describes the pending review", () => { + const o = valid("tasks.completion-with-gate"); const reviewTask = (o as unknown as { reviewTask: Record }).reviewTask; + Object.assign(reviewTask, { status: "done", executionState: { ...(reviewTask.executionState as object), status: "completed" } }); expect(gradeInternal(o).passed).toBe(false); }); + it("accepts a product continuation run of Morgan's implementation", () => { + const o = valid("tasks.completion-with-gate"); + o.runs.push({ id: "continuation", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" } }); + (o as unknown as { attachments: Array<{ originatingRunId: string }> }).attachments[0]!.originatingRunId = "continuation"; + expect(gradeInternal(o).passed).toBe(true); + o.runs.push({ id: "failed", companyId: "company", agentId: "morgan", status: "failed", contextSnapshot: { issueId: "task" } }); + expect(gradeInternal(o).passed).toBe(false); + }); +}); + +describe("completion gate setup readiness", () => { + const review = { status: "pending", currentStageId: "stage", currentStageType: "review", currentParticipant: { type: "user", userId: "riley" } }; + const ready = () => ({ task: { id: "task", status: "in_review", executionState: structuredClone(review) }, + runs: [{ id: "run", agentId: "morgan", status: "succeeded" }], + attachments: [{ id: "html", sha256: "a", createdByAgentId: "morgan", originatingRunId: "run" }] }); + it("accepts the configured review pending after a delivered implementation", () => { + expect(unmetPriorDeliveryReview(ready(), "morgan", "riley")).toEqual([]); + const continued = ready(); continued.runs.push({ id: "next", agentId: "morgan", status: "succeeded" }); + expect(unmetPriorDeliveryReview(continued, "morgan", "riley")).toEqual([]); + }); + it("names each unmet field of a review requested outside the configured stage", () => { + const generic = ready(); Object.assign(generic.task, { executionState: null }); + expect(unmetPriorDeliveryReview(generic, "morgan", "riley")).toEqual([ + "executionState.status=null (want pending)", "executionState.currentParticipant=null (want the configured reviewer)"]); + }); + it("names a bypassed review, an active run and a missing delivery", () => { + const bypassed = ready(); Object.assign(bypassed.task, { status: "done", executionState: null }); + bypassed.runs.push({ id: "live", agentId: "morgan", status: "running" }); bypassed.attachments = []; + expect(unmetPriorDeliveryReview(bypassed, "morgan", "riley")).toEqual([ + "task.status=done (want in_review)", "executionState.status=null (want pending)", + "executionState.currentParticipant=null (want the configured reviewer)", "active worker runs=1 (want 0)", + "worker deliveries from a succeeded worker run=0 of 0 attachments (want at least 1)"]); + const other = ready(); other.task.executionState.currentParticipant.userId = "someone"; + expect(unmetPriorDeliveryReview(other, "morgan", "riley")).toEqual(["executionState.currentParticipant=user:other (want the configured reviewer)"]); + }); }); describe("private delegation audience calibration", () => {