mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): accept bound hire revision chat history
This commit is contained in:
1 parent
c6cc59282a
commit
cf58269631
4 files changed
+176
-18
No files matched your search
@@ -4209,7 +4209,7 @@
|
||||
"chat-rendered-links.ts",
|
||||
"chat-scenarios/questions-and-approvals.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v3",
|
||||
"oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v4",
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 300000,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
const clock = vi.hoisted(() => ({ now: 30_000, deadlines: [] as number[] }));
|
||||
const observations = vi.hoisted(() => ({ read: vi.fn() }));
|
||||
vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal<typeof import("./identity-support.js")>(), observeIdentity: observations.read }));
|
||||
vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value }));
|
||||
vi.mock("@playwright/test", () => ({ expect: (node: { visible?: () => boolean; text?: () => string }) => ({
|
||||
toBeVisible: async () => { if (!node.visible?.()) throw Error("Actual pending proposal is not visible"); },
|
||||
@@ -17,8 +19,8 @@ vi.mock("../api.js", () => ({ pollUntil: async (options: { deadlineAt: number; l
|
||||
throw Error("Controlled original observation expired");
|
||||
} }));
|
||||
import { hireRevisionIdentityCases } from "./identity-hire-revision.js";
|
||||
afterEach(() => { vi.restoreAllMocks(); document.body.replaceChildren(); });
|
||||
type Options = { deadline?: number | "missing"; readyAt?: number; href?: string; wrongComment?: boolean; foreignSource?: boolean; wrongActor?: boolean; wrongNativeComment?: boolean; wrongModel?: boolean; notPending?: boolean; lateDom?: boolean };
|
||||
afterEach(() => { vi.restoreAllMocks(); observations.read.mockReset(); document.body.replaceChildren(); });
|
||||
type Options = { deadline?: number | "missing"; readyAt?: number; href?: string; wrongComment?: boolean; foreignSource?: boolean; wrongActor?: boolean; wrongNativeComment?: boolean; wrongModel?: boolean; notPending?: boolean; lateDom?: boolean; progress?: boolean; genericReply?: boolean };
|
||||
function fixture(options: Options = {}) {
|
||||
clock.now = 30_000; clock.deadlines.splice(0); vi.spyOn(Date, "now").mockImplementation(() => clock.now);
|
||||
vi.spyOn(Element.prototype, "getClientRects").mockImplementation(() => [{ width: 10, height: 10 }] as unknown as DOMRectList);
|
||||
@@ -27,10 +29,11 @@ function fixture(options: Options = {}) {
|
||||
const readyAt = options.readyAt ?? 210_000, href = options.href ?? "/approvals/approval";
|
||||
const run = { id: "native", companyId: "company", agentId: options.wrongActor ? "other" : "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: source, commentId: options.wrongNativeComment ? "other" : "human" } };
|
||||
const comments = [{ id: "human", companyId: "company", issueId: source, authorUserId: "person", body: query },
|
||||
{ id: "reply", companyId: "company", issueId: source, authorAgentId: "ceo", createdByRunId: "native", body: "Revised pending proposal." }];
|
||||
...(options.progress ? [{ id: "progress", companyId: "company", issueId: source, authorAgentId: "ceo", createdByRunId: "native", body: "I am checking this proposal before revising it." }] : []),
|
||||
{ id: "reply", companyId: "company", issueId: source, authorAgentId: "ceo", createdByRunId: "native", body: options.genericReply ? "Revised and resubmitted for approval." : "Revised to requested-model, pending proposal." }];
|
||||
function sync() {
|
||||
document.body.innerHTML = '<section data-testid="task-chat-thread"><div id="comment-human"></div><div id="comment-' + (options.wrongComment ? "foreign" : "reply") + '"></div></section>';
|
||||
if (clock.now >= readyAt) { const a = document.createElement("a"); a.setAttribute("href", href); a.textContent = "pending approval"; document.querySelector('div:last-child')!.append(a); }
|
||||
document.body.innerHTML = '<section data-testid="task-chat-thread"><div id="comment-human"></div>' + (options.progress ? '<div id="comment-progress"></div>' : '') + '<div id="comment-' + (options.wrongComment ? "foreign" : "reply") + '"></div></section>';
|
||||
if (clock.now >= readyAt) { const a = document.createElement("a"); a.setAttribute("href", href); a.textContent = "pending approval"; document.querySelector('div:last-child')!.append(a); if (options.progress) document.getElementById("comment-progress")!.append(a.cloneNode(true)); }
|
||||
}
|
||||
type Nodes = { nodes: Element[]; locator(selector: string): Nodes; first(): Nodes; filter(options: { has: Nodes }): Nodes; evaluateAll<T>(read: (nodes: Element[]) => T): Promise<T>; click(): Promise<void>; text(): string };
|
||||
function nodes(list: Element[]): Nodes { return { nodes: list, locator: selector => nodes(list.flatMap(n => Array.from(n.querySelectorAll(selector)))), first: () => nodes(list.slice(0,1)),
|
||||
@@ -57,7 +60,9 @@ function fixture(options: Options = {}) {
|
||||
it("awaits the delayed genuine canonical-reply review link within original300s, clicks and inspects actual pending revised model", async () => {
|
||||
const f = fixture(); await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state);
|
||||
expect(f.clicks()).toBe(1); expect(clock.now).toBe(210_000); expect(clock.deadlines.every(x => x === 300_000)).toBe(true);
|
||||
expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true }); expect(f.url()).toContain("/chats/ceo");
|
||||
expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true,
|
||||
revisionHistoryComment: { id: "reply", authorAgentId: "ceo", createdByRunId: "native" },
|
||||
revisionHistoryLink: { source: "reply", conversationId: "source", commentId: "reply", href: "/approvals/approval" } }); expect(f.url()).toContain("/chats/ceo");
|
||||
});
|
||||
it.each([{ href: "https://foreign.test/approvals/approval" }, { href: "/OTHER/approvals/approval" }, { href: "/approvals/other" }, { wrongComment: true }, { foreignSource: true }, { wrongActor: true }, { wrongNativeComment: true }, { readyAt: 300_001 }, { lateDom: true }, { deadline: "missing" }, { deadline: NaN }, { deadline: 29_999 }])("rejects absent/foreign/late review readiness before any click %j", async options => {
|
||||
const f = fixture(options as Options); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks()).toBe(0);
|
||||
@@ -68,3 +73,38 @@ it("accepts the same owned company-prefixed ordinary approval route", async () =
|
||||
it.each([{ wrongModel: true }, { notPending: true }])("does not turn review visibility into a wrong model/approved proposal PASS %j", async options => {
|
||||
const f = fixture({ readyAt:30_000, ...options }); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx,f.state)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("captures the model-specific bound revision note when progress also renders the same proposal link", async () => {
|
||||
const f = fixture({ readyAt: 30_000, progress: true });
|
||||
await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state);
|
||||
expect(f.state).toMatchObject({ revisionHistoryComment: { id: "reply", body: "Revised to requested-model, pending proposal." },
|
||||
revisionHistoryLink: { commentId: "reply", href: "/approvals/approval" } });
|
||||
});
|
||||
|
||||
it("rereads the actual source comment independently of the captured history receipt", async () => {
|
||||
const captured = { id: "reply", issueId: "source", companyId: "company", createdByRunId: "native", authorAgentId: "ceo", body: "Revised model, pending approval." };
|
||||
const observed = structuredClone(captured), child = { id: "child-comment", issueId: "child", body: "Child history." };
|
||||
const state = { sourceIssueId: "source", originalApproval: { id: "approval", payload: { adapterConfig: { model: "old" } } }, revisionHistoryComment: captured };
|
||||
observations.read.mockResolvedValue({ comments: [child], state });
|
||||
const get = vi.fn(async (url: string) => {
|
||||
if (url === "/api/approvals/approval") return { id: "approval", payload: { adapterConfig: { model: "new" } } };
|
||||
if (url === "/api/approvals/approval/comments") return [];
|
||||
if (url === "/api/issues/source/comments?order=asc") return [observed];
|
||||
throw Error("Unexpected observation read");
|
||||
});
|
||||
const ctx = { input: { api: { get } } } as unknown as ChatCatalogContext;
|
||||
const result = await hireRevisionIdentityCases[0]!.observe!(ctx, state) as { comments: typeof captured[] };
|
||||
expect(get).toHaveBeenCalledWith("/api/issues/source/comments?order=asc");
|
||||
expect(result.comments).toEqual([child, captured]);
|
||||
expect(result.comments[1]).not.toBe(captured);
|
||||
observed.body = "Changed independent observation";
|
||||
expect(captured.body).toBe("Revised model, pending approval.");
|
||||
});
|
||||
|
||||
it("navigates a generic owned approval link without requiring the chat message to repeat the model", async () => {
|
||||
const f = fixture({ readyAt: 30_000, genericReply: true });
|
||||
await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state);
|
||||
expect(f.clicks()).toBe(1);
|
||||
expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true,
|
||||
revisionHistoryComment: { id: "reply", body: "Revised and resubmitted for approval." } });
|
||||
});
|
||||
@@ -11,7 +11,90 @@ function valid(): IdentityObservation {
|
||||
originalModel: "openrouter/deepseek/deepseek-v4-pro", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true,
|
||||
approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } };
|
||||
}
|
||||
function validChatRevision(): IdentityObservation {
|
||||
const o = valid(), query = "Change the proposed hire to the cheaper model and send it back for approval.";
|
||||
o.state.approvalComments = (o.state.approvalComments as unknown[]).slice(0, 1);
|
||||
o.state.sourceIssueId = "source";
|
||||
o.reply = "Updated the proposed hire to the cheaper model `openrouter/deepseek/deepseek-v4.1-flash-1017` (was `deepseek-v4-pro`) and resubmitted it. It's awaiting your review at [the pending approval](/approvals/approval) — it must be approved before the hire proceeds.";
|
||||
o.humanQueries = [query];
|
||||
o.runs = [{ id: "native", companyId: "acorn", agentId: "ceo", status: "succeeded", runtimeMode: "native", nativeIssueId: "source",
|
||||
resultJson: { finalizationPhase: "committed" }, contextSnapshot: { source: "issue.comment", issueId: "source", commentId: "human",
|
||||
paperclipWake: { comments: [{ id: "human", issueId: "source", body: query, author: { type: "user", id: "alex" } }] } } }];
|
||||
const comment = { id: "bound-reply", companyId: "acorn", issueId: "source", authorAgentId: "ceo", authorUserId: null, createdByRunId: "native", body: o.reply };
|
||||
o.comments = [structuredClone(comment)];
|
||||
o.state.revisionHistoryComment = structuredClone(comment);
|
||||
Object.assign(o.company, { issuePrefix: "ACORN" });
|
||||
o.renderedLinkEvidence = [{ threadTestId: "task-chat-thread", source: "reply", href: "/approvals/approval", conversationId: "source", anchorCommentId: "bound-reply", commentId: "bound-reply" }];
|
||||
o.state.revisionHistoryLink = structuredClone(o.renderedLinkEvidence[0]);
|
||||
return o;
|
||||
}
|
||||
describe("formal hire remains a proposal after cost revision", () => {
|
||||
it("accepts retained original history and the exact bound native revision reply without an extra approval comment", () => {
|
||||
expect(gradeHireRevision(validChatRevision()).passed).toBe(true);
|
||||
});
|
||||
it("accepts a generic linked chat message only when the owned approval note supplies model-specific history", () => {
|
||||
const o = validChatRevision(), body = "Revised and resubmitted for approval. [Pending proposal](/approvals/approval).";
|
||||
o.state.approvalComments = valid().state.approvalComments;
|
||||
o.reply = body;
|
||||
o.comments[0]!.body = body;
|
||||
(o.state.revisionHistoryComment as { body: string }).body = body;
|
||||
expect(gradeHireRevision(o).passed).toBe(true);
|
||||
const missing = structuredClone(o);
|
||||
missing.state.approvalComments = (missing.state.approvalComments as unknown[]).slice(0, 1);
|
||||
expect(gradeHireRevision(missing).passed).toBe(false);
|
||||
const foreign = structuredClone(o);
|
||||
(foreign.state.approvalComments as Array<{ authorAgentId?: string }>)[1]!.authorAgentId = "other";
|
||||
expect(gradeHireRevision(foreign).passed).toBe(false);
|
||||
});
|
||||
it("accepts a bound revision reply alongside progress from the same run", () => {
|
||||
const o = validChatRevision();
|
||||
const progress = { ...structuredClone(o.comments[0]!), id: "progress", body: "I am checking the revised proposal." };
|
||||
o.comments.unshift(progress);
|
||||
o.reply = o.comments.map(row => row.body).join("\n\n");
|
||||
expect(gradeHireRevision(o).passed).toBe(true);
|
||||
});
|
||||
it("accepts an earlier bound revision note when the final message only asks for board review", () => {
|
||||
const o = validChatRevision();
|
||||
o.comments.push({ ...structuredClone(o.comments[0]!), id: "final", body: "It remains pending board approval." });
|
||||
o.reply = o.comments.map(row => row.body).join("\n\n");
|
||||
expect(gradeHireRevision(o).passed).toBe(true);
|
||||
});
|
||||
it("keeps captured comment and link receipts separate from independently observed evidence", () => {
|
||||
const o = validChatRevision();
|
||||
expect(o.state.revisionHistoryComment).not.toBe(o.comments[0]);
|
||||
expect(o.state.revisionHistoryLink).not.toBe(o.renderedLinkEvidence![0]);
|
||||
const observed = structuredClone({ comments: o.comments, links: o.renderedLinkEvidence });
|
||||
(o.state.revisionHistoryComment as { body: string }).body = "Changed captured receipt only";
|
||||
(o.state.revisionHistoryLink as { href: string }).href = "/ACORN/approvals/approval";
|
||||
expect({ comments: o.comments, links: o.renderedLinkEvidence }).toEqual(observed);
|
||||
expect(gradeHireRevision(o).passed).toBe(false);
|
||||
});
|
||||
it.each([
|
||||
["unobserved captured message", (o: IdentityObservation) => { o.comments = []; }],
|
||||
["unbound prose", (o: IdentityObservation) => { delete o.state.revisionHistoryComment; }],
|
||||
["wrong comment company", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { companyId: string }).companyId = "other"; }],
|
||||
["wrong comment source", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { issueId: string }).issueId = "other"; }],
|
||||
["wrong comment actor", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { authorAgentId: string }).authorAgentId = "other"; }],
|
||||
["wrong originating run", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { createdByRunId: string }).createdByRunId = "other"; }],
|
||||
["wrong model in reply", (o: IdentityObservation) => { o.reply = o.reply.replace(String(o.state.cheaperModel), "other-model"); (o.state.revisionHistoryComment as { body: string }).body = o.reply; }],
|
||||
["uncommitted run", (o: IdentityObservation) => { o.runs[0]!.resultJson = { finalizationPhase: "pending" }; }],
|
||||
["foreign run company", (o: IdentityObservation) => { o.runs[0]!.companyId = "other"; }],
|
||||
["foreign run actor", (o: IdentityObservation) => { o.runs[0]!.agentId = "other"; }],
|
||||
["foreign run source", (o: IdentityObservation) => { o.runs[0]!.nativeIssueId = "other"; }],
|
||||
["wrong human wake", (o: IdentityObservation) => { o.runs[0]!.contextSnapshot!.commentId = "other"; }],
|
||||
["unobserved rendered link", (o: IdentityObservation) => { o.renderedLinkEvidence = []; }],
|
||||
["different canonical receipt href", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { href: string }).href = "/ACORN/approvals/approval"; }],
|
||||
["wrong captured link source", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { source: string }).source = "project-created-card"; }],
|
||||
["wrong captured link conversation", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { conversationId: string }).conversationId = "other"; }],
|
||||
["wrong captured link comment", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { commentId: string }).commentId = "other"; }],
|
||||
["changed captured message", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { body: string }).body += " Unobserved receipt text"; }],
|
||||
["wrong proposal link", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { href: string }).href = "/approvals/other"; }],
|
||||
["foreign link origin", (o: IdentityObservation) => { (o.state.revisionHistoryLink as { href: string }).href = "https://other.test/approvals/approval"; }],
|
||||
["missing original history", (o: IdentityObservation) => { o.state.approvalComments = []; }],
|
||||
["duplicate resubmit", (o: IdentityObservation) => { o.activity.push({ ...o.activity[1]!, id: "duplicate" }); }],
|
||||
["approved proposal", (o: IdentityObservation) => { (o.state.currentApproval as { status: string }).status = "approved"; }],
|
||||
["created employee", (o: IdentityObservation) => { o.agents.push({ id: "new-hire" }); }],
|
||||
] as const)("rejects bound chat history with %s", (_name, mutate) => { const o = validChatRevision(); mutate(o); expect(gradeHireRevision(o).passed).toBe(false); });
|
||||
it("requires actual model choice, attributed history and a still pending proposal", () => { expect(gradeHireRevision(valid()).passed).toBe(true); expect(gradeHireRevision(undefined).passed).toBe(false); });
|
||||
it("rejects treating requested revision as approval or activating the hire", () => { const o = valid(); (o.state.currentApproval as { status: string }).status = "approved"; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.agents.push({ id: "new-hire" }); expect(gradeHireRevision(wrong).passed).toBe(false); });
|
||||
it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; expect(gradeHireRevision(o).passed).toBe(false); });
|
||||
|
||||
@@ -2,12 +2,39 @@ import { expect } from "@playwright/test";
|
||||
import type { Approval, ApprovalComment } from "../../../packages/shared/src/types/approval.js";
|
||||
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { collectConversationLinks, conversationLinkLocator } from "../chat-rendered-links.js";
|
||||
import { collectConversationLinks, conversationLinkLocator, type RenderedConversationLink } from "../chat-rendered-links.js";
|
||||
import { checks, entity, type Entity } from "./internal-support.js";
|
||||
import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
|
||||
import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply, type IdentityObservation } from "./identity-support.js";
|
||||
import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js";
|
||||
import { readHireModelChoice, validHireModelChoice } from "./identity-hire-model-choice.js";
|
||||
|
||||
function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | undefined): boolean {
|
||||
const comment = o.state.revisionHistoryComment as Entity | undefined;
|
||||
const link = o.state.revisionHistoryLink as RenderedConversationLink | undefined;
|
||||
const sourceId = o.state.sourceIssueId;
|
||||
if (!approval || !comment || !link || typeof sourceId !== "string" || o.runs.length !== 1 || o.humanQueries.length !== 1
|
||||
|| comment.companyId !== o.companyId || comment.issueId !== sourceId || comment.authorAgentId !== o.leadId || comment.authorUserId
|
||||
|| typeof comment.body !== "string" || typeof o.state.cheaperModel !== "string"
|
||||
|| !comment.body.includes(o.state.cheaperModel)) return false;
|
||||
const run = o.runs[0]!;
|
||||
if (run.id !== comment.createdByRunId || run.companyId !== o.companyId || run.agentId !== o.leadId || run.runtimeMode !== "native"
|
||||
|| run.status !== "succeeded" || run.resultJson?.finalizationPhase !== "committed" || run.nativeIssueId !== sourceId
|
||||
|| run.contextSnapshot?.source !== "issue.comment" || run.contextSnapshot.issueId !== sourceId) return false;
|
||||
if (!o.comments.some(row => row.id === comment.id && row.companyId === o.companyId && row.issueId === sourceId
|
||||
&& row.authorAgentId === o.leadId && !row.authorUserId && row.createdByRunId === run.id && row.body === comment.body)) return false;
|
||||
const wake = run.contextSnapshot.paperclipWake as { comments?: Array<{ id?: string; issueId?: string; body?: string; author?: { type?: string; id?: string } }> } | undefined;
|
||||
if (!wake?.comments?.some(row => row.id === run.contextSnapshot?.commentId && row.issueId === sourceId
|
||||
&& row.author?.type === "user" && row.author.id === o.state.humanId && row.body?.trim() === o.humanQueries[0]!.trim())) return false;
|
||||
if (link.source !== "reply" || link.conversationId !== sourceId || link.commentId !== comment.id
|
||||
|| !o.renderedLinkEvidence?.some(row => row.source === "reply" && row.conversationId === sourceId
|
||||
&& row.commentId === comment.id && row.href === link.href)) return false;
|
||||
try {
|
||||
const url = new URL(link.href, o.origin);
|
||||
const prefix = (o.company as Entity & { issuePrefix?: string }).issuePrefix;
|
||||
return url.origin === new URL(o.origin).origin && (url.pathname === `/approvals/${approval.id}`
|
||||
|| !!prefix && url.pathname === `/${prefix}/approvals/${approval.id}`);
|
||||
} catch { return false; }
|
||||
}
|
||||
export function gradeHireRevision(value: unknown) {
|
||||
const c = checks(); const o = identityObservation(value); c.add("complete-formal-revision", !!o, "Actual governance proposal, catalog choice, saved history and attributed resubmission are required."); if (!o) return c.result();
|
||||
const approval = o.state.currentApproval as Approval | undefined; const original = o.state.originalApproval as Approval | undefined;
|
||||
@@ -25,10 +52,11 @@ export function gradeHireRevision(value: unknown) {
|
||||
const comments = o.state.approvalComments as ApprovalComment[] | undefined;
|
||||
c.add("inspectable-original-and-revised-history", !!comments?.some(comment => comment.id === o.state.originalHistoryCommentId && comment.authorUserId === o.state.humanId
|
||||
&& comment.body.includes(String(o.state.originalModel)) && comment.body.includes(String(o.state.cheaperModel)))
|
||||
&& comments.some(comment => comment.authorAgentId === o.leadId && comment.body.includes(String(o.state.cheaperModel)))
|
||||
&& (comments.some(comment => comment.authorAgentId === o.leadId && comment.body.includes(String(o.state.cheaperModel)))
|
||||
|| hasBoundRevisionHistory(o, approval))
|
||||
&& o.activity.some(row => row.action === "approval.revision_requested" && row.entityId === approval?.id && row.actorType === "user")
|
||||
&& o.activity.filter(row => row.action === "approval.resubmitted" && row.entityId === approval?.id && row.actorType === "agent" && row.agentId === o.leadId).length === 1,
|
||||
"The retained public original/board-choice comment, agent revision note and actual revision/resubmission audit form inspectable history.");
|
||||
"The retained public original/board-choice comment, agent approval note or bound native revision reply, and actual revision/resubmission audit form inspectable history.");
|
||||
c.add("no-hire-or-approval-side-effect", o.agents.length === o.state.beforeAgentCount && !o.activity.some(row => row.action === "approval.approved" && row.entityId === approval?.id)
|
||||
&& o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true && /approval|review|pending|board/i.test(o.reply),
|
||||
"The actual proposal remains visibly pending with the revised model and no employee is instantiated or activated under the revision request.");
|
||||
@@ -54,7 +82,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe
|
||||
return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id,
|
||||
eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get<unknown[]>(`/api/companies/${fixtures.company.id}/agents`)).length };
|
||||
}, gradeHireRevision, {
|
||||
authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"],
|
||||
authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v4", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"],
|
||||
async act(ctx, state) {
|
||||
const remaining = () => remainingIdentityObservationBudget(ctx);
|
||||
remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining();
|
||||
@@ -76,25 +104,30 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe
|
||||
const bound = ctx.issue()?.id === sourceId && ctx.issue()?.conversationState === "waiting" && runs.length === 1 && !!run
|
||||
&& run.companyId === ctx.input.fixtures.company.id && run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"
|
||||
&& run.contextSnapshot?.issueId === sourceId && humans.length === 1 && run.contextSnapshot?.commentId === humans[0]?.id;
|
||||
const link = bound && run ? links.find(row => {
|
||||
const boundLinks = bound && run ? links.filter(row => {
|
||||
if (row.source !== "reply" || row.conversationId !== sourceId || !comments.some(comment => comment.id === row.commentId
|
||||
&& comment.authorAgentId === run.agentId && comment.createdByRunId === run.id)) return false;
|
||||
try { const url = new URL(row.href, ctx.input.api.baseURL); return url.origin === new URL(ctx.input.api.baseURL).origin
|
||||
&& [ `/approvals/${id}`, `/${ctx.input.fixtures.company.issuePrefix}/approvals/${id}` ].includes(url.pathname); }
|
||||
catch { return false; }
|
||||
}) : undefined;
|
||||
return { approval, link };
|
||||
}) : [];
|
||||
const link = boundLinks.find(row => comments.some(comment => comment.id === row.commentId
|
||||
&& comment.body?.includes(String(state.cheaperModel)))) ?? boundLinks[0];
|
||||
const comment = link ? comments.find(row => row.id === link.commentId) : undefined;
|
||||
return { approval, link, comment };
|
||||
},
|
||||
accept: value => Date.now() < ctx.observationDeadlineAt! && value.approval.id === id && value.approval.companyId === ctx.input.fixtures.company.id
|
||||
&& value.approval.status === "pending" && !!value.link,
|
||||
&& value.approval.status === "pending" && !!value.link && !!value.comment,
|
||||
reject: value => value.approval.id !== id || value.approval.companyId !== ctx.input.fixtures.company.id || value.approval.status !== "pending"
|
||||
? "The owned revised hire proposal is not pending" : undefined,
|
||||
});
|
||||
remaining(); if (!frame.link) throw new Error("Actual pending revised proposal review link is missing");
|
||||
remaining(); if (!frame.link || !frame.comment) throw new Error("Actual pending revised proposal review link is missing");
|
||||
await conversationLinkLocator(ctx.input.page, frame.link).first().click({ timeout: remaining() });
|
||||
await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible({ timeout: remaining() }); state.pendingProposalVisible = true;
|
||||
await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click({ timeout: remaining() });
|
||||
await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel), { timeout: remaining() }); state.revisedModelVisible = true;
|
||||
state.revisionHistoryComment = frame.comment;
|
||||
state.revisionHistoryLink = frame.link;
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
@@ -103,6 +136,8 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe
|
||||
state.approvalComments = await ctx.input.api.get<ApprovalComment[]>(`/api/approvals/${id}/comments`);
|
||||
const stripModel = (payload: Record<string, unknown>) => { const adapter = payload.adapterConfig as Record<string, unknown>; return { ...payload, adapterConfig: { ...adapter, model: undefined } }; };
|
||||
state.originalOtherFieldsPreserved = JSON.stringify(stripModel((state.originalApproval as Approval).payload)) === JSON.stringify(stripModel(current.payload));
|
||||
return observeIdentity(ctx, state);
|
||||
const sourceComments = await ctx.input.api.get<Entity[]>(`/api/issues/${String(state.sourceIssueId)}/comments?order=asc`);
|
||||
const observation = await observeIdentity(ctx, state);
|
||||
return { ...observation, comments: [...observation.comments.filter(row => row.issueId !== state.sourceIssueId), ...sourceComments] };
|
||||
},
|
||||
})];
|
||||
Reference in new issue
Block a user