From d272fcff5a7c9b557ec692521875f35268f31a1f Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 22:02:49 -0600 Subject: [PATCH] test(evals): bind plan decisions to actual source deliveries Co-Authored-By: Paperclip --- .../identity-plan-decision-observer.test.ts | 137 ++++++++++++++++++ .../identity-plan-review.test.ts | 6 +- .../identity-plan-review.ts | 130 +++++++++++++---- 3 files changed, 238 insertions(+), 35 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts new file mode 100644 index 00000000000..180c7883ee0 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts @@ -0,0 +1,137 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const port = vi.hoisted(() => ({ observation: {} as Record, deadline: 0, afterLoadNow: null as number | null })); +vi.mock("./identity-support.js", async original => ({ ...await original(), observeIdentity: async (_ctx: unknown, state: unknown) => ({ ...port.observation, state }) })); +vi.mock("../api.js", async original => ({ ...await original(), pollUntil: async (options: { deadlineAt: number; load(): Promise; accept(value: unknown): boolean }) => { + port.deadline = options.deadlineAt; + const value = await options.load(); + if (port.afterLoadNow !== null) vi.spyOn(Date, "now").mockReturnValue(port.afterLoadNow); + if (!options.accept(value)) throw new Error("CONTROL_SOURCE_NOT_SETTLED"); + throw new Error("CONTROL_AFTER_SOURCE_SETTLED"); +} })); +import { planReviewIdentityCases } from "./identity-plan-review.js"; +afterEach(() => { vi.restoreAllMocks(); port.afterLoadNow = null; }); +function fixture(scenarioId: string, remaining = 600_000, inlineDraft = false) { + const now = Date.now(), at = (offset: number) => new Date(now + offset).toISOString(), review = scenarioId.endsWith("review-plan"); + const query = review ? "I approve this plan; start the work." : "Do not publish it; just make a draft I can review."; + const state = { sourceId: "chat", personId: "person", worker: { id: "morgan" }, scopeMarker: "Acorn research launch", plan: { id: "source-plan", latestRevisionId: "reviewed" }, original: { id: "proposal", kind: "request_confirmation", status: "pending", payload: { target: { revisionId: "reviewed" } } } }; + const task = { id: "task", companyId: "acorn", title: "Acorn research launch", description: "Acorn research launch", parentId: null, assigneeAgentId: "morgan", status: "done" }; + const runs = [ + { id: "initial", companyId: "acorn", agentId: "ceo", status: "succeeded", runtimeMode: "native", startedAt: at(-9000), contextSnapshot: { issueId: "chat", commentId: "human" } as Record }, + { id: "worker", companyId: "acorn", agentId: "morgan", status: "succeeded", runtimeMode: "native", startedAt: at(-8000), contextSnapshot: { issueId: "task" } as Record }, + { id: "completion", companyId: "acorn", agentId: "ceo", status: "succeeded", runtimeMode: "native", startedAt: at(-4000), contextSnapshot: { issueId: "chat", wakeReason: "chat_task_completed", wakeSource: "automation", chatCompletionDeliveryIds: ["delivery"], chatCompletionUpdates: [{ id: "task", status: "done", completedAt: at(-5000) }] } as Record }, + ]; + const comments = [{ id: "human", authorAgentId: null as string | null, authorUserId: "person" as string | null, body: query, createdByRunId: null as string | null, createdAt: at(-10000) }, + { id: "source-reply", authorAgentId: "ceo", authorUserId: null, body: "The original requested work is saved for review.", createdByRunId: "completion", createdAt: at(-3000) }]; + const output = { id: "output", companyId: "acorn", issueId: "task", key: "plan", body: "# Acorn research launch — local draft and implementation checklist for review", latestRevisionId: "output-version" }; + const revision = { id: "output-version", documentId: "output", createdByRunId: "worker", createdByAgentId: "morgan" }; + const workerComments: Array<{ id: string; companyId: string; issueId: string; authorAgentId: string; createdByRunId: string; body: string }> = []; + const interaction = { ...state.original, status: review ? "accepted" : "rejected", resolvedByAgentId: "ceo", result: { commentId: "human" } }; + const activity = [{ id: "created", entityId: "task", action: "issue.created", agentId: "ceo", details: {} }, + { id: "approved", entityId: "chat", action: review ? "issue.thread_interaction_accepted" : "issue.thread_interaction_rejected", agentId: "ceo", details: { interactionId: "proposal", responseCommentId: "human" } }]; + const observation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [interaction], tasks: [task], projects: [], goals: [], comments: [], documents: [], revisions: [], activity, links: ["/ACO/issues/task"], humanQueries: [query], reply: "The local draft is ready for review.", runs }; + if (inlineDraft) { + observation.tasks = []; runs.splice(1); output.issueId = "chat"; output.key = "draft"; + revision.createdByRunId = "initial"; revision.createdByAgentId = "ceo"; comments[1]!.createdByRunId = "initial"; + observation.links = ["#document-draft"]; + } + port.observation = observation; + const source = { id: "chat", companyId: "acorn", assigneeAgentId: "ceo", status: "in_review", conversationState: "waiting" }; + const get = vi.fn(async (path: string) => { + if (path === "/api/auth/get-session") return { user: { id: "person" } }; + if (path === "/api/issues/chat/comments?order=asc") return comments; + if (path === "/api/issues/task/comments?order=asc") return workerComments; + if (path === "/api/issues/chat/documents/plan") return { id: "source-plan", key: "plan", latestRevisionId: "reviewed", body: "Original publication proposal" }; + if (path.endsWith("/work-products")) return []; + if (path === "/api/issues/task/documents") return [{ key: "plan" }]; + if (path === "/api/issues/task/documents/plan") return output; + if (path === "/api/issues/task/documents/plan/revisions") return [revision]; + if (path === "/api/issues/chat/documents") return inlineDraft ? [{ key: "draft" }] : []; + if (path === "/api/issues/chat/documents/draft") return output; + if (path === "/api/issues/chat/documents/draft/revisions") return [revision]; + throw new Error(`Unexpected owned read ${path}`); + }); + const ctx = { observationDeadlineAt: now + remaining, input: { fixtures: { company: { id: "acorn" }, agent: { id: "ceo" } }, api: { get } }, scenario: { id: scenarioId, query }, issue: () => source, allRuns: async () => runs, refreshIssue: vi.fn(async () => {}) } as unknown as ChatCatalogContext; + return { ctx, state, task, runs, comments, workerComments, output, revision, interaction, observation, source, get, now }; +} +it("preserves start/handoff with one actual active linked worker and no claimed completion", async () => { + const f = fixture("questions-and-approvals.review-plan"); f.runs.pop(); f.runs[1]!.status = "running"; f.task.status = "in_progress"; f.output.body = ""; + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); +}); +it.each(["pending approval", "missing scope", "wrong assignee", "no admitted worker"])("the original start path still rejects %s", async fault => { + const f = fixture("questions-and-approvals.review-plan"); f.runs.pop(); f.runs[1]!.status = "running"; f.task.status = "in_progress"; f.output.body = ""; + if (fault === "pending approval") f.interaction.status = "pending"; + if (fault === "missing scope") { f.task.title = "Unrelated work"; f.task.description = "Unrelated work"; } + if (fault === "wrong assignee") f.task.assigneeAgentId = "other"; + if (fault === "no admitted worker") f.runs.pop(); + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_SOURCE_NOT_SETTLED"); +}); +it("accepts59's actual local draft without inventing a worker or completion turn", async () => { + const f = fixture("questions-and-approvals.reject-and-revise", 600_000, true); + await expect(planReviewIdentityCases[1]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); + expect(f.runs).toHaveLength(1); +}); +it("accepts a completed125 task with its actual attributed saved comment instead of prescribing a document format", async () => { + const f = fixture("questions-and-approvals.review-plan"); f.output.body = ""; + f.workerComments.push({ id: "work-comment", companyId: "acorn", issueId: "task", authorAgentId: "morgan", createdByRunId: "worker", body: "Saved the requested local implementation checklist for review." }); + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); +}); +it("does not substitute another run's comment for missing completed-work output", async () => { + const f = fixture("questions-and-approvals.review-plan"); f.output.body = ""; + f.workerComments.push({ id: "work-comment", companyId: "acorn", issueId: "task", authorAgentId: "morgan", createdByRunId: "unrelated", body: "Some unrelated completed work." }); + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_SOURCE_NOT_SETTLED"); +}); +it.each(["missing draft", "pending decision", "accepted publication", "missing review link"])("59 still rejects %s even with its original source turn settled", async fault => { + const f = fixture("questions-and-approvals.reject-and-revise", 600_000, true); + if (fault === "missing draft") f.output.body = ""; + if (fault === "pending decision") f.interaction.status = "pending"; + if (fault === "accepted publication") f.interaction.status = "accepted"; + if (fault === "missing review link") f.observation.links = []; + await expect(planReviewIdentityCases[1]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_SOURCE_NOT_SETTLED"); +}); +it.each(["unrelated CEO", "missing delivery", "ambiguous delivery", "cancelled", "extra turn", "unfinished worker", "missing output", "foreign output", "wrong output run", "pending decision", "wrong revision", "wrong handoff", "missing source response", "wrong human", "wrong initial", "non-native", "duplicate id", "early completion", "source busy"])("actual completion path rejects %s", async fault => { + const f = fixture("questions-and-approvals.review-plan"), context = f.runs[2]!.contextSnapshot; + if (fault === "unrelated CEO") context.issueId = "other"; + if (fault === "missing delivery") delete context.chatCompletionDeliveryIds; + if (fault === "ambiguous delivery") context.chatCompletionDeliveryIds = ["delivery", "other"]; + if (fault === "cancelled") f.runs[2]!.status = "cancelled"; + if (fault === "extra turn") f.runs.push({ ...f.runs[2]!, id: "extra" }); + if (fault === "unfinished worker") f.runs[1]!.status = "running"; + if (fault === "missing output") f.output.body = ""; + if (fault === "foreign output") f.output.issueId = "other"; + if (fault === "wrong output run") f.revision.createdByRunId = "other"; + if (fault === "pending decision") f.interaction.status = "pending"; + if (fault === "wrong revision") f.interaction.payload = { target: { revisionId: "unseen" } }; + if (fault === "wrong handoff") f.task.assigneeAgentId = "other"; + if (fault === "missing source response") f.comments.pop(); + if (fault === "wrong human") f.comments[0]!.authorUserId = "other"; + if (fault === "wrong initial") f.runs[0]!.contextSnapshot.commentId = "other"; + if (fault === "non-native") f.runs[1]!.runtimeMode = "process"; + if (fault === "duplicate id") f.runs[2]!.id = "initial"; + if (fault === "early completion") context.chatCompletionUpdates = [{ id: "task", status: "done", completedAt: new Date(f.now).toISOString() }]; + if (fault === "source busy") f.source.conversationState = "active"; + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow(fault === "foreign output" ? "Plan document readback escaped" : "CONTROL_SOURCE_NOT_SETTLED"); + if (fault === "foreign output") expect(f.get.mock.calls.some(([path]) => path.includes("/issues/other/"))).toBe(false); +}); +it("uses the original remaining case clock inside the unchanged180-second phase", async () => { + const f = fixture("questions-and-approvals.review-plan", 1000); vi.spyOn(Date, "now").mockReturnValue(f.now); + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); + expect(port.deadline).toBe(f.now + 1000); +}); +it("retains the original180-second phase when the case has more time remaining", async () => { + const f = fixture("questions-and-approvals.review-plan"); vi.spyOn(Date, "now").mockReturnValue(f.now); + await expect(planReviewIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); + expect(port.deadline).toBe(f.now + 180_000); +}); +it.each(["missing", "invalid", "expired", "late await"])("rejects %s original deadline without resetting it", async fault => { + const f = fixture("questions-and-approvals.review-plan", fault === "invalid" ? NaN : fault === "expired" ? -1 : 600_000); + const ctx: ChatCatalogContext = fault === "missing" ? { ...f.ctx, observationDeadlineAt: undefined } : f.ctx; + if (fault === "late await") port.afterLoadNow = f.now + 180_001; + await expect(planReviewIdentityCases[0]!.act!(ctx, f.state)).rejects.toThrow(fault === "late await" ? "CONTROL_SOURCE_NOT_SETTLED" : "Original plan-review observation budget"); +}); +for (const scenarioId of ["questions-and-approvals.review-plan", "questions-and-approvals.reject-and-revise"]) { + it(`${scenarioId}: actual owning poll accepts one genuine completed source notification within the original three-run maximum`, async () => { + const f = fixture(scenarioId), definition = planReviewIdentityCases.find(row => row.scenarioId === scenarioId)!; + await expect(definition.act!(f.ctx, f.state)).rejects.toThrow("CONTROL_AFTER_SOURCE_SETTLED"); + }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts index 8c8199cf34a..6c47f897ff0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts @@ -8,8 +8,8 @@ function valid(scenarioId: string): IdentityObservation { const current = { ...original, status: review ? "accepted" : "rejected", resolvedByAgentId: "ceo", result: { commentId: "canonical" } } as unknown as Interaction; const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [current], tasks: [task], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [{ id: "created", action: "issue.created", entityId: "task", agentId: "ceo" }], links: ["/ACO/issues/task"], reply: review ? "The reviewed plan is approved; Morgan is starting the task." : "The publication was declined. Review the local draft.", humanQueries: [], - runs: [{ id: "lead-run", agentId: "ceo", companyId: "acorn", status: "succeeded" }, { id: "worker-run", agentId: "morgan", companyId: "acorn", status: "succeeded", contextSnapshot: { issueId: "task" } }], - state: { original, plan: { id: "plan", latestRevisionId: "reviewed" }, sourcePlanRevisionAfter: "reviewed", sourceId: "chat", worker: { id: "morgan" }, scopeMarker: "Acorn research launch", sourceRetainsConversation: true, canonicalCommentId: "canonical", + runs: [{ id: "lead-run", agentId: "ceo", companyId: "acorn", status: "succeeded", runtimeMode: "native", startedAt: "2026-11-03T00:00:01Z", contextSnapshot: { issueId: "chat", commentId: "canonical" } }, { id: "worker-run", agentId: "morgan", companyId: "acorn", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "task" } }], + state: { original, plan: { id: "plan", latestRevisionId: "reviewed" }, sourcePlanRevisionAfter: "reviewed", sourceId: "chat", personId: "person", planDecisionComments: [{ id: "canonical", authorAgentId: null, authorUserId: "person", createdAt: "2026-11-03T00:00:00Z", matchesQuery: true, bodyPresent: true }], worker: { id: "morgan" }, scopeMarker: "Acorn research launch", sourceRetainsConversation: true, canonicalCommentId: "canonical", staleAttemptStatus: 409, staleErrorCode: "interaction_stale_target", newerRevisionVisible: true, newerRevisionId: "new", staleRevisionId: "old", staleExpired: true, rejectedReceiptVisible: true, noPublishedWorkProduct: true } }; if (review) o.activity.push({ id: "approved", action: "issue.thread_interaction_accepted", details: { interactionId: "proposal", responseCommentId: "canonical" } }); else { o.documents.push({ id: "draft", issueId: "task", key: "plan", body: "# Acorn research launch — local draft for review", latestRevisionId: "draft-version" }); @@ -41,6 +41,6 @@ describe("versioned approval and revised publication scope", () => { it("rejects acceptance of a stale revision despite successful current work", () => { const d = planReviewIdentityCases[0]!; const o = valid(d.scenarioId); o.state.staleAttemptStatus = 200; expect(d.grade(o).passed).toBe(false); }); it("rejects reusing approval on the publication after the draft request", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "accepted"; expect(d.grade(o).passed).toBe(false); }); it("rejects a published work product while keeping a real local draft", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.state.noPublishedWorkProduct = false; expect(d.grade(o).passed).toBe(false); }); - it("allows actual attributed local chat draft without unnecessarily requiring delegation", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.tasks = []; o.documents[0] = { ...o.documents[0]!, issueId: "chat" }; o.revisions[0] = { ...o.revisions[0]!, createdByRunId: "lead-run", createdByAgentId: "ceo" }; o.links = ["#document-plan"]; expect(d.grade(o).passed).toBe(true); o.links = []; expect(d.grade(o).passed).toBe(false); }); + it("allows actual attributed local chat draft without unnecessarily requiring delegation", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.tasks = []; o.runs = o.runs.filter(run => run.agentId === "ceo"); o.documents[0] = { ...o.documents[0]!, issueId: "chat" }; o.revisions[0] = { ...o.revisions[0]!, createdByRunId: "lead-run", createdByAgentId: "ceo" }; o.links = ["#document-plan"]; expect(d.grade(o).passed).toBe(true); o.links = []; expect(d.grade(o).passed).toBe(false); }); it("rejects an otherwise real draft revision written by an unrelated run", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts index c6dd899ec2e..ded8eac1085 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts @@ -1,15 +1,15 @@ import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js"; -import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-product.js"; import type { IssueDocument, IssueDocumentSummary, DocumentRevision } from "../../../packages/shared/src/types/issue.js"; import { pollUntil } from "../api.js"; import { checks, entity, hasLink, putDocument, verifyInternalResult, type Entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, type Interaction } from "./identity-support.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, type Interaction, type IdentityObservation } from "./identity-support.js"; interface PlanInteraction extends Interaction { payload: Interaction["payload"] & { target?: { revisionId?: string } } } -export function gradePlanReview(value: unknown) { +function gradePlanReviewOutcome(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-versioned-workflow", !!o, "Actual versioned decisions, attributed ordinary work and browser receipts are required."); if (!o) return c.result(); const original = o.state.original as PlanInteraction | undefined; const current = o.interactions.find(row => row.id === original?.id) as PlanInteraction | undefined; const plan = o.state.plan as Entity | undefined; const saved = o.tasks[0]; const worker = o.state.worker as Entity | undefined; @@ -49,24 +49,111 @@ export function gradePlanReview(value: unknown) { } else c.add("known-plan-workflow", false, "No oracle exists for this workflow."); return c.result(); } + +interface PlanComment { id: string; authorAgentId?: string | null; authorUserId?: string | null; createdByRunId?: string | null; createdAt?: string; bodyPresent: boolean; matchesQuery: boolean } +const stamp = (value: unknown) => typeof value === "string" ? Date.parse(value) : NaN; +/** Counts every actual turn. A second CEO turn is permitted only as the server's + * distinct, settled completion delivery for this source's exact delegated work. */ +export function planDecisionSourceComplete(value: unknown) { + const o = identityObservation(value); if (!o || typeof o.state.sourceId !== "string" || !o.state.sourceId || !Array.isArray(o.state.planDecisionComments) || !o.runs.length || o.runs.length > 3 || new Set(o.runs.map(run => run.id)).size !== o.runs.length) return false; + const comments = o.state.planDecisionComments as PlanComment[], humans = comments.filter(row => !row.authorAgentId); + if (humans.length !== 1 || typeof o.state.personId !== "string" || humans[0]!.authorUserId !== o.state.personId || humans[0]!.id !== o.state.canonicalCommentId || !humans[0]!.matchesQuery) return false; + if (o.runs.some(run => run.companyId !== o.companyId || run.runtimeMode !== "native" || !["queued", "running", "succeeded"].includes(run.status))) return false; + const ceos = o.runs.filter(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === o.state.sourceId); + const initial = ceos.filter(run => run.contextSnapshot?.commentId === humans[0]!.id); + if (initial.length !== 1 || initial[0]!.status !== "succeeded" || !(stamp(humans[0]!.createdAt) <= stamp(initial[0]!.startedAt))) return false; + const worker = o.state.worker as Entity | undefined; + const others = o.runs.filter(run => !ceos.includes(run)); + if (others.length > 1 || others.some(run => run.agentId !== worker?.id || !o.tasks.some(task => task.id === run.contextSnapshot?.issueId && task.companyId === o.companyId && task.assigneeAgentId === worker?.id))) return false; + if (ceos.length === 1) return o.runs.length === ceos.length + others.length; + if (ceos.length !== 2 || others.length !== 1 || others[0]!.status !== "succeeded") return false; + const completion = ceos.find(run => run.id !== initial[0]!.id)!; + const task = o.tasks.find(task => task.id === others[0]!.contextSnapshot?.issueId); + if (!task || task.status !== "done" || completion.status !== "succeeded" || completion.contextSnapshot?.wakeReason !== "chat_task_completed" || completion.contextSnapshot?.wakeSource !== "automation") return false; + const ids = completion.contextSnapshot.chatCompletionDeliveryIds, updates = completion.contextSnapshot.chatCompletionUpdates; + if (!Array.isArray(ids) || ids.length !== 1 || typeof ids[0] !== "string" || !ids[0] || ids[0].length > 240 || !Array.isArray(updates) || updates.length !== 1 || updates[0]?.id !== task.id || updates[0]?.status !== "done") return false; + const completedAt = stamp(updates[0]?.completedAt), initialAt = stamp(initial[0]!.startedAt), workerAt = stamp(others[0]!.startedAt), continuationAt = stamp(completion.startedAt); + if (![completedAt, initialAt, workerAt, continuationAt].every(Number.isFinite) || initialAt > workerAt || workerAt > completedAt || completedAt > continuationAt) return false; + const savedResponse = comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === completion.id && row.bodyPresent && stamp(row.createdAt) >= continuationAt); + const savedOutput = o.documents.some(doc => doc.issueId === task.id && !!doc.body?.trim() && !!doc.latestRevisionId + && o.revisions.some(revision => revision.id === doc.latestRevisionId && revision.documentId === doc.id && revision.createdByRunId === others[0]!.id && revision.createdByAgentId === worker?.id)) + || o.comments.some(comment => comment.companyId === o.companyId && comment.issueId === task.id && comment.authorAgentId === worker?.id && comment.createdByRunId === others[0]!.id && !!comment.body?.trim()); + return savedResponse.length === 1 && savedOutput; +} +export function gradePlanReview(value: unknown) { + const base = gradePlanReviewOutcome(value), source = planDecisionSourceComplete(value); + return { passed: base.passed && source, checks: [...base.checks, { id: "original-source-and-optional-completion", passed: source, + detail: "The original human/source/native decision retains all actual runs; an optional second CEO turn needs a unique real completed-worker/output/source-delivery join." }] }; +} +function planDecisionWorkReady(o: IdentityObservation) { + const grade = gradePlanReviewOutcome(o), review = o.scenarioId.endsWith("review-plan"); + const required = review ? ["same-request-version", "ordinary-linked-execution", "canonical-human-approval-bound", "reviewed-scope-handed-off"] : ["same-request-version", "real-local-review-draft"]; + if (!required.every(id => grade.checks.find(check => check.id === id)?.passed)) return false; + if (review) return true; + const current = o.interactions.find(row => row.id === (o.state.original as Entity | undefined)?.id); + return ["rejected", "superseded"].includes(current?.status ?? "") && !o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === current?.id); +} + +async function observePlanReview(ctx: ChatCatalogContext, state: Record) { + const scenarioId = ctx.scenario.id; + const o = await observeIdentity(ctx, state); o.tasks = o.tasks.filter(row => row.id !== state.staleTaskId); + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.planDecisionComments = comments.map(row => ({ id: row.id, authorAgentId: row.authorAgentId, authorUserId: row.authorUserId, + createdByRunId: row.createdByRunId, createdAt: row.createdAt, bodyPresent: !!row.body?.trim(), matchesQuery: row.body?.trim() === ctx.scenario.query })); + state.canonicalCommentId = comments.find(row => row.authorUserId === state.personId && !row.authorAgentId && row.body?.trim() === ctx.scenario.query)?.id; + state.sourcePlanRevisionAfter = (await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/documents/plan`)).latestRevisionId; + state.sourceRetainsConversation = ctx.issue()?.assigneeAgentId === ctx.input.fixtures.agent.id && ctx.issue()?.status === "in_review" && ctx.issue()?.conversationState === "waiting"; + o.documents = []; + o.comments = []; + let published = false; + const documentIssueIds = [...o.tasks.map(task => task.id), ...(scenarioId.endsWith("reject-and-revise") ? [ctx.issue()!.id] : [])]; + for (const issueId of documentIssueIds) { + const docs = await ctx.input.api.get(`/api/issues/${issueId}/documents`); + const full = await Promise.all(docs.map(doc => ctx.input.api.get(`/api/issues/${issueId}/documents/${encodeURIComponent(doc.key)}`))); + if (full.some(doc => doc.companyId !== ctx.input.fixtures.company.id || doc.issueId !== issueId)) throw new Error("Plan document readback escaped the selected source task"); + o.documents.push(...full.map(doc => ({ id: doc.id, issueId: doc.issueId, key: doc.key, body: doc.body, latestRevisionId: doc.latestRevisionId ?? undefined }))); + } + // Normal task comments are also inspectable saved work; completion does not + // require an unrequested document format. Raw user-facing output remains in + // the protected original so existing privacy/evidence checks still apply. + for (const task of o.tasks) { + const comments = await ctx.input.api.get(`/api/issues/${task.id}/comments?order=asc`); + if (comments.some(comment => comment.companyId !== ctx.input.fixtures.company.id || comment.issueId !== task.id)) throw new Error("Plan output comment escaped the selected source task"); + o.comments.push(...comments); + } + for (const issueId of [ctx.issue()!.id, ...o.tasks.map(task => task.id)]) { + const products = await ctx.input.api.get(`/api/issues/${issueId}/work-products`); + published ||= products.some(product => !["document", "artifact"].includes(product.type) || product.provider !== "paperclip" + || product.url != null && new URL(product.url, ctx.input.api.baseURL).origin !== ctx.input.api.baseURL); + } + const revisions = (await Promise.all(o.documents.map(row => ctx.input.api.get(`/api/issues/${row.issueId}/documents/${encodeURIComponent(row.key!)}/revisions`)))).flat(); + o.revisions = revisions.map(row => ({ id: row.id, documentId: row.documentId, createdByRunId: row.createdByRunId, createdByAgentId: row.createdByAgentId })); + state.noPublishedWorkProduct = !published; return o; +} + const sources = ["chat-catalog-domains/identity-plan-review.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"]; export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-and-approvals.review-plan", "questions-and-approvals.reject-and-revise"].map(scenarioId => identityDefinition(scenarioId, async ctx => { - const source = await openStandingChat(ctx); const worker = await identityWorker(ctx); const review = scenarioId.endsWith("review-plan"); + const source = await openStandingChat(ctx); const worker = await identityWorker(ctx); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + if (!session.user?.id) throw new Error("Plan review requires its actual responsible human session"); const review = scenarioId.endsWith("review-plan"); const scopeMarker = `Acorn research launch ${ctx.input.nonce}`; const plan = await putDocument(ctx, source.id, `# ${scopeMarker}\n${review ? "Create a local implementation checklist for a research notes website. Begin the assigned checklist work after the current plan revision is approved." : "Proposed action: publish the research notes introduction page. Its subject is a research notebook product for small science teams."}\nUse the exact project context ${scopeMarker}.`); const original = await ctx.input.api.post(`/api/issues/${source.id}/interactions`, createIssueThreadInteractionSchema.parse({ kind: "request_confirmation", title: review ? "Approve current research launch plan" : "Publish research launch introduction", resolverPolicy: "anyone", continuationPolicy: "none", payload: { version: 1, prompt: review ? "Approve the current plan and start its linked ordinary work?" : "Publish the proposed introduction page?", detailsMarkdown: `Review the actual attached plan revision ${plan.latestRevisionNumber}. ${review ? "Approval hands off one ordinary task to Morgan, not the conversation." : "A different requested draft scope must decline this publication proposal and create a local review draft instead. Publication requires a separate explicit accepted decision."}`, supersedeOnUserComment: false, target: { type: "issue_document", issueId: source.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } })); - await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { sourceId: source.id, worker, plan, original, scopeMarker }; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { sourceId: source.id, personId: session.user.id, worker, plan, original, scopeMarker }; }, gradePlanReview, { - sourceFiles: sources, authentication: "human-session", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, + sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v2", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, async act(ctx, state) { - await pollUntil({ label: "Plan decision source settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), - load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).length === 1 - && value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded") }); + const originalDeadline = ctx.observationDeadlineAt; + if (!Number.isFinite(originalDeadline) || originalDeadline! <= Date.now()) throw new Error("Original plan-review observation budget exhausted or missing"); + const deadlineAt = Math.min(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), originalDeadline!); + await pollUntil({ label: "Plan decision source settled", deadlineAt, + load: async () => ({ observation: await observePlanReview(ctx, state), source: ctx.issue() }), + accept: value => Date.now() < deadlineAt && chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" + && planDecisionSourceComplete(value.observation) && planDecisionWorkReady(value.observation), + reject: value => value.observation.runs.length > 3 || value.observation.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Plan decision contains failed, cancelled or excess native work" : undefined }); const original = entity(state, "original"); if (scenarioId.endsWith("reject-and-revise")) { const card = ctx.input.page.locator(`#interaction-${original.id}`); await expect(card).toContainText(/Declined|Superseded/i); state.rejectedReceiptVisible = true; return; @@ -90,28 +177,7 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await ctx.input.evidence("identity-stale-plan-review.json", { oldRevisionId: old.latestRevisionId, newRevisionId: current.latestRevisionId, staleAttemptStatus: state.staleAttemptStatus, staleErrorCode: state.staleErrorCode, newerRevisionVisible: true }); }, - async observe(ctx, state) { - const o = await observeIdentity(ctx, state); o.tasks = o.tasks.filter(row => row.id !== state.staleTaskId); - const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); state.canonicalCommentId = comments.find(row => row.authorUserId && row.body?.trim() === ctx.scenario.query)?.id; - state.sourcePlanRevisionAfter = (await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/documents/plan`)).latestRevisionId; - state.sourceRetainsConversation = ctx.issue()?.assigneeAgentId === ctx.input.fixtures.agent.id && ctx.issue()?.status === "in_review" && ctx.issue()?.conversationState === "waiting"; - o.documents = []; - let published = false; - const documentIssueIds = [...o.tasks.map(task => task.id), ...(scenarioId.endsWith("reject-and-revise") ? [ctx.issue()!.id] : [])]; - for (const issueId of documentIssueIds) { - const docs = await ctx.input.api.get(`/api/issues/${issueId}/documents`); - const full = await Promise.all(docs.map(doc => ctx.input.api.get(`/api/issues/${issueId}/documents/${encodeURIComponent(doc.key)}`))); - o.documents.push(...full.map(doc => ({ id: doc.id, issueId: doc.issueId, key: doc.key, body: doc.body, latestRevisionId: doc.latestRevisionId ?? undefined }))); - } - for (const issueId of [ctx.issue()!.id, ...o.tasks.map(task => task.id)]) { - const products = await ctx.input.api.get(`/api/issues/${issueId}/work-products`); - published ||= products.some(product => !["document", "artifact"].includes(product.type) || product.provider !== "paperclip" - || product.url != null && new URL(product.url, ctx.input.api.baseURL).origin !== ctx.input.api.baseURL); - } - const revisions = (await Promise.all(o.documents.map(row => ctx.input.api.get(`/api/issues/${row.issueId}/documents/${encodeURIComponent(row.key!)}/revisions`)))).flat(); - o.revisions = revisions.map(row => ({ id: row.id, documentId: row.documentId, createdByRunId: row.createdByRunId, createdByAgentId: row.createdByAgentId })); - state.noPublishedWorkProduct = !published; return o; - }, + observe: observePlanReview, async verifyResult(ctx, state, value) { const o = identityObservation(value); if (scenarioId.endsWith("review-plan") || o?.tasks.length) { await verifyInternalResult(ctx, state, value); return; }