From 2a7d9b029b0332e0f31b0164dab975e43fc4562e Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 16:34:23 -0600 Subject: [PATCH] test(evals): accept bound hire revision chat history --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-hire-review-readiness.test.ts | 4 +- .../identity-hire-revision.test.ts | 37 +++++++++++++++ .../identity-hire-revision.ts | 45 +++++++++++++++---- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index c73285ce97d..57bcc9c9cae 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4209,7 +4209,7 @@ "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v3", + "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v4", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts index fc66db7a827..6c1e7449707 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts @@ -57,7 +57,9 @@ function fixture(options: Options = {}) { it("awaits the delayed genuine canonical-reply review link within original300s, clicks and inspects actual pending revised model", async () => { const f = fixture(); await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state); expect(f.clicks()).toBe(1); expect(clock.now).toBe(210_000); expect(clock.deadlines.every(x => x === 300_000)).toBe(true); - expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true }); expect(f.url()).toContain("/chats/ceo"); + expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true, + revisionHistoryComment: { id: "reply", authorAgentId: "ceo", createdByRunId: "native" }, + revisionHistoryLink: { source: "reply", conversationId: "source", commentId: "reply", href: "/approvals/approval" } }); expect(f.url()).toContain("/chats/ceo"); }); it.each([{ href: "https://foreign.test/approvals/approval" }, { href: "/OTHER/approvals/approval" }, { href: "/approvals/other" }, { wrongComment: true }, { foreignSource: true }, { wrongActor: true }, { wrongNativeComment: true }, { readyAt: 300_001 }, { lateDom: true }, { deadline: "missing" }, { deadline: NaN }, { deadline: 29_999 }])("rejects absent/foreign/late review readiness before any click %j", async options => { const f = fixture(options as Options); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks()).toBe(0); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts index bdcc4b774a0..83c81d0fcb9 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts @@ -11,7 +11,44 @@ function valid(): IdentityObservation { originalModel: "openrouter/deepseek/deepseek-v4-pro", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true, approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } }; } +function validChatRevision(): IdentityObservation { + const o = valid(), query = "Change the proposed hire to the cheaper model and send it back for approval."; + o.state.approvalComments = (o.state.approvalComments as unknown[]).slice(0, 1); + o.state.sourceIssueId = "source"; + o.reply = "Updated the proposed hire to the cheaper model `openrouter/deepseek/deepseek-v4.1-flash-1017` (was `deepseek-v4-pro`) and resubmitted it. It's awaiting your review at [the pending approval](/approvals/approval) — it must be approved before the hire proceeds."; + o.humanQueries = [query]; + o.runs = [{ id: "native", companyId: "acorn", agentId: "ceo", status: "succeeded", runtimeMode: "native", nativeIssueId: "source", + resultJson: { finalizationPhase: "committed" }, contextSnapshot: { source: "issue.comment", issueId: "source", commentId: "human", + paperclipWake: { comments: [{ id: "human", issueId: "source", body: query, author: { type: "user", id: "alex" } }] } } }]; + o.state.revisionHistoryComment = { id: "bound-reply", companyId: "acorn", issueId: "source", authorAgentId: "ceo", authorUserId: null, createdByRunId: "native", body: o.reply }; + o.renderedLinkEvidence = [{ threadTestId: "task-chat-thread", source: "reply", href: "/approvals/approval", conversationId: "source", anchorCommentId: "bound-reply", commentId: "bound-reply" }]; + o.state.revisionHistoryLink = o.renderedLinkEvidence[0]; + return o; +} describe("formal hire remains a proposal after cost revision", () => { + it("accepts retained original history and the exact bound native revision reply without an extra approval comment", () => { + expect(gradeHireRevision(validChatRevision()).passed).toBe(true); + }); + it.each([ + ["unbound prose", (o: IdentityObservation) => { delete o.state.revisionHistoryComment; }], + ["wrong comment company", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { companyId: string }).companyId = "other"; }], + ["wrong comment source", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { issueId: string }).issueId = "other"; }], + ["wrong comment actor", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { authorAgentId: string }).authorAgentId = "other"; }], + ["wrong originating run", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { createdByRunId: string }).createdByRunId = "other"; }], + ["wrong model in reply", (o: IdentityObservation) => { o.reply = o.reply.replace(String(o.state.cheaperModel), "other-model"); (o.state.revisionHistoryComment as { body: string }).body = o.reply; }], + ["uncommitted run", (o: IdentityObservation) => { o.runs[0]!.resultJson = { finalizationPhase: "pending" }; }], + ["foreign run company", (o: IdentityObservation) => { o.runs[0]!.companyId = "other"; }], + ["foreign run actor", (o: IdentityObservation) => { o.runs[0]!.agentId = "other"; }], + ["foreign run source", (o: IdentityObservation) => { o.runs[0]!.nativeIssueId = "other"; }], + ["wrong human wake", (o: IdentityObservation) => { o.runs[0]!.contextSnapshot!.commentId = "other"; }], + ["unobserved rendered link", (o: IdentityObservation) => { o.renderedLinkEvidence = []; }], + ["wrong proposal link", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { href: string }).href = "/approvals/other"; }], + ["foreign link origin", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { href: string }).href = "https://other.test/approvals/approval"; }], + ["missing original history", (o: IdentityObservation) => { o.state.approvalComments = []; }], + ["duplicate resubmit", (o: IdentityObservation) => { o.activity.push({ ...o.activity[1]!, id: "duplicate" }); }], + ["approved proposal", (o: IdentityObservation) => { (o.state.currentApproval as { status: string }).status = "approved"; }], + ["created employee", (o: IdentityObservation) => { o.agents.push({ id: "new-hire" }); }], + ] as const)("rejects bound chat history with %s", (_name, mutate) => { const o = validChatRevision(); mutate(o); expect(gradeHireRevision(o).passed).toBe(false); }); it("requires actual model choice, attributed history and a still pending proposal", () => { expect(gradeHireRevision(valid()).passed).toBe(true); expect(gradeHireRevision(undefined).passed).toBe(false); }); it("rejects treating requested revision as approval or activating the hire", () => { const o = valid(); (o.state.currentApproval as { status: string }).status = "approved"; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.agents.push({ id: "new-hire" }); expect(gradeHireRevision(wrong).passed).toBe(false); }); it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; expect(gradeHireRevision(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index 4a13046572c..d6d14ca5bc1 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -2,12 +2,37 @@ import { expect } from "@playwright/test"; import type { Approval, ApprovalComment } from "../../../packages/shared/src/types/approval.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; -import { collectConversationLinks, conversationLinkLocator } from "../chat-rendered-links.js"; +import { collectConversationLinks, conversationLinkLocator, type RenderedConversationLink } from "../chat-rendered-links.js"; import { checks, entity, type Entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply } from "./identity-support.js"; +import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply, type IdentityObservation } from "./identity-support.js"; import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js"; import { readHireModelChoice, validHireModelChoice } from "./identity-hire-model-choice.js"; +function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | undefined): boolean { + const comment = o.state.revisionHistoryComment as Entity | undefined; + const link = o.state.revisionHistoryLink as RenderedConversationLink | undefined; + const sourceId = o.state.sourceIssueId; + if (!approval || !comment || !link || typeof sourceId !== "string" || o.runs.length !== 1 || o.humanQueries.length !== 1 + || comment.companyId !== o.companyId || comment.issueId !== sourceId || comment.authorAgentId !== o.leadId || comment.authorUserId + || typeof comment.body !== "string" || comment.body !== o.reply || typeof o.state.cheaperModel !== "string" + || !comment.body.includes(o.state.cheaperModel)) return false; + const run = o.runs[0]!; + if (run.id !== comment.createdByRunId || run.companyId !== o.companyId || run.agentId !== o.leadId || run.runtimeMode !== "native" + || run.status !== "succeeded" || run.resultJson?.finalizationPhase !== "committed" || run.nativeIssueId !== sourceId + || run.contextSnapshot?.source !== "issue.comment" || run.contextSnapshot.issueId !== sourceId) return false; + const wake = run.contextSnapshot.paperclipWake as { comments?: Array<{ id?: string; issueId?: string; body?: string; author?: { type?: string; id?: string } }> } | undefined; + if (!wake?.comments?.some(row => row.id === run.contextSnapshot?.commentId && row.issueId === sourceId + && row.author?.type === "user" && row.author.id === o.state.humanId && row.body?.trim() === o.humanQueries[0]!.trim())) return false; + if (link.source !== "reply" || link.conversationId !== sourceId || link.commentId !== comment.id + || !o.renderedLinkEvidence?.some(row => row.source === "reply" && row.conversationId === sourceId + && row.commentId === comment.id && row.href === link.href)) return false; + try { + const url = new URL(link.href, o.origin); + const prefix = (o.company as Entity & { issuePrefix?: string }).issuePrefix; + return url.origin === new URL(o.origin).origin && (url.pathname === `/approvals/${approval.id}` + || !!prefix && url.pathname === `/${prefix}/approvals/${approval.id}`); + } catch { return false; } +} export function gradeHireRevision(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-formal-revision", !!o, "Actual governance proposal, catalog choice, saved history and attributed resubmission are required."); if (!o) return c.result(); const approval = o.state.currentApproval as Approval | undefined; const original = o.state.originalApproval as Approval | undefined; @@ -25,10 +50,11 @@ export function gradeHireRevision(value: unknown) { const comments = o.state.approvalComments as ApprovalComment[] | undefined; c.add("inspectable-original-and-revised-history", !!comments?.some(comment => comment.id === o.state.originalHistoryCommentId && comment.authorUserId === o.state.humanId && comment.body.includes(String(o.state.originalModel)) && comment.body.includes(String(o.state.cheaperModel))) - && comments.some(comment => comment.authorAgentId === o.leadId && comment.body.includes(String(o.state.cheaperModel))) + && (comments.some(comment => comment.authorAgentId === o.leadId && comment.body.includes(String(o.state.cheaperModel))) + || hasBoundRevisionHistory(o, approval)) && o.activity.some(row => row.action === "approval.revision_requested" && row.entityId === approval?.id && row.actorType === "user") && o.activity.filter(row => row.action === "approval.resubmitted" && row.entityId === approval?.id && row.actorType === "agent" && row.agentId === o.leadId).length === 1, - "The retained public original/board-choice comment, agent revision note and actual revision/resubmission audit form inspectable history."); + "The retained public original/board-choice comment, agent approval note or bound native revision reply, and actual revision/resubmission audit form inspectable history."); c.add("no-hire-or-approval-side-effect", o.agents.length === o.state.beforeAgentCount && !o.activity.some(row => row.action === "approval.approved" && row.entityId === approval?.id) && o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true && /approval|review|pending|board/i.test(o.reply), "The actual proposal remains visibly pending with the revised model and no employee is instantiated or activated under the revision request."); @@ -54,7 +80,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v4", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining(); @@ -83,18 +109,21 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe && [ `/approvals/${id}`, `/${ctx.input.fixtures.company.issuePrefix}/approvals/${id}` ].includes(url.pathname); } catch { return false; } }) : undefined; - return { approval, link }; + const comment = link ? comments.find(row => row.id === link.commentId) : undefined; + return { approval, link, comment }; }, accept: value => Date.now() < ctx.observationDeadlineAt! && value.approval.id === id && value.approval.companyId === ctx.input.fixtures.company.id - && value.approval.status === "pending" && !!value.link, + && value.approval.status === "pending" && !!value.link && !!value.comment, reject: value => value.approval.id !== id || value.approval.companyId !== ctx.input.fixtures.company.id || value.approval.status !== "pending" ? "The owned revised hire proposal is not pending" : undefined, }); - remaining(); if (!frame.link) throw new Error("Actual pending revised proposal review link is missing"); + remaining(); if (!frame.link || !frame.comment) throw new Error("Actual pending revised proposal review link is missing"); await conversationLinkLocator(ctx.input.page, frame.link).first().click({ timeout: remaining() }); await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible({ timeout: remaining() }); state.pendingProposalVisible = true; await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click({ timeout: remaining() }); await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel), { timeout: remaining() }); state.revisedModelVisible = true; + state.revisionHistoryComment = frame.comment; + state.revisionHistoryLink = frame.link; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining(); }, async observe(ctx, state) {