Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-07 07:55:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): wait for settled primary Chat evidence

Browse Source 
Require the actual recent CEO conversation, original visible human and native reply, and editable composer after navigation and final profile proof. Re-read the persisted Morgan preference and exact original chat/run set after readiness. Use only remaining original case budget and preserve all existing primary/privacy grading. Calibrate the actual final verifier with loading, late side-effect, wrong identity and exhausted-budget negatives.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 20:58:49 -06:00
1 parent b7dfdbbefd
commit 462af87e84
2 files changed
+105 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts
+55
View File
@@ -0,0 +1,55 @@
import { afterEach, expect, it, vi } from "vitest";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
vi.mock("@playwright/test", () => ({ expect: (target: any) => ({
toBeVisible: async () => { if (!await target.visible()) throw Error("Actual Chat is still loading or its conversation is absent"); },
toBeEditable: async () => { if (!target.editable()) throw Error("Chat composer is not ready"); },
toHaveURL: async (matches: (url: URL) => boolean) => { if (!matches(new URL(target.url()))) throw Error("Wrong selected Chat route"); },
}) }));
import { runtimePrimaryCases } from "./runtime-primary.js";
afterEach(() => { vi.restoreAllMocks(); });
function fixture(options: { loadingForever?: boolean; extraRun?: boolean; extraChat?: boolean; wrongPrimary?: boolean; wrongPerson?: boolean;
wrongChat?: boolean; noReply?: boolean; wrongRequest?: boolean; missingRenderedReply?: boolean; readOnly?: boolean; expired?: boolean; lateReady?: boolean } = {}) {
const deadline = Date.now() + (options.expired ? -1 : 30_000); let current = "https://fixture.test/RUN/chats/ceo", ready = false;
const calls: string[] = [];
const source = { id: "source", companyId: "company", conversationAgentId: "ceo", conversationUserId: "person" };
const comments = [{ id: "human", authorUserId: "person", body: "Make Morgan my go-to agent." },
...(options.noReply ? [] : [{ id: "reply", authorAgentId: "ceo", createdByRunId: "native", body: "Use the human primary control." }])];
const primary = { companyId: "company", userId: options.wrongPerson ? "foreign" : "person", primaryAgentId: options.wrongPrimary ? "ceo" : "morgan" };
const locator = (kind: string) => ({ first() { return this; }, locator: () => locator("editor"),
visible: async () => {
calls.push(kind);
if (kind === "composer") {
if (options.loadingForever) return false; ready = true;
if (options.lateReady) vi.spyOn(Date, "now").mockReturnValue(deadline + 1);
}
return kind === "heading" || ready && !(kind.includes("comment-reply") && options.missingRenderedReply);
}, editable: () => ready && !options.readOnly });
const page = { url: () => current, goto: async (url: string) => { current = new URL(url, "https://fixture.test").href; ready = false; },
reload: async () => {}, getByRole: () => locator("heading"), getByTestId: () => locator("composer"), locator };
const api = { baseURL: "https://fixture.test", get: async (path: string) => {
calls.push(path); if (path.endsWith("/primary-agent/me")) return primary;
if (path.endsWith("/chats/ceo")) return { ...source, ...(options.wrongChat ? { id: "different-source" } : {}) };
if (path.includes("/comments?")) return comments.map(row => options.wrongRequest && row.id === "human" ? { ...row, body: "different query" } : row);
if (path.endsWith("/chats")) return [source, ...(options.extraChat && ready ? [{ ...source, id: "extra" }] : [])];
throw Error("Unexpected API read");
} };
const state = { worker: { id: "morgan", name: "Morgan", companyId: "company" }, primaryBefore: { companyId: "company", userId: "person", primaryAgentId: "ceo" },
sourceBeforeNavigation: "source", nativeIdsBeforeNavigation: ["native"], chatCountBeforeNavigation: 1 };
const evidence = vi.fn(async (_name: string, _value: unknown) => {});
const ctx = { observationDeadlineAt: deadline, scenario: { id: "agents.choose-primary", query: "Make Morgan my go-to agent." },
input: { api, page, evidence, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } } },
issue: () => source, refreshIssue: async () => {}, allRuns: async () => [{ id: "native", agentId: "ceo", status: "succeeded" }, ...(options.extraRun && ready ? [{ id: "unexpected", agentId: "ceo", status: "running" }] : [])],
} as unknown as ChatCatalogContext;
const observation = { companyId: "company", leadId: "ceo", state, reply: "Use the human primary control.", agents: [{ id: "ceo", name: "CEO" }, state.worker], tasks: [], links: [], activity: [], runs: [], humanQueries: [] };
return { ctx, state, observation, evidence, calls, ready: () => ready };
}
it("actual final verifier waits for meaningful selected Chat then rechecks preference and no extra work before capture", async () => {
const f = fixture(); await runtimePrimaryCases[0]!.verifyResult(f.ctx, f.state, f.observation);
expect(f.ready()).toBe(true); expect(f.evidence).toHaveBeenCalledWith("chat-primary-ready-conversation.json", expect.objectContaining({ sourceId: "source", agentId: "ceo", primaryAgentId: "morgan", noExtraWork: true }));
expect(f.calls).toContain('[id="comment-human"]'); expect(f.calls).toContain('[id="comment-reply"]');
});
it.each([{ loadingForever: true }, { extraRun: true }, { extraChat: true }, { wrongPrimary: true }, { wrongPerson: true }, { wrongChat: true },
{ noReply: true }, { wrongRequest: true }, { missingRenderedReply: true }, { readOnly: true }, { expired: true }, { lateReady: true }])("actual final verifier rejects incomplete or changed settled state %j", async options => {
const f = fixture(options); await expect(runtimePrimaryCases[0]!.verifyResult(f.ctx, f.state, f.observation)).rejects.toThrow();
expect(f.evidence.mock.calls.some(row => row[0] === "chat-primary-ready-conversation.json")).toBe(false);
});
tests/runner-e2e/chat-catalog-domains/runtime-primary.ts
+50 -6
View File
@@ -1,8 +1,8 @@
import { expect } from "@playwright/test";
import { checks, type Entity } from "./internal-support.js";
import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js";
import { probeAgentPermission, waitForReply } from "./identity-support.js";
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js";
import { probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js";
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, verifyRuntimeResult, type RuntimeAgent } from "./runtime-support.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
interface Primary { companyId: string; userId: string; primaryAgentId: string | null }
type Person = Awaited<ReturnType<typeof createAuthenticatedPerson>>;
@@ -35,6 +35,44 @@ export function gradePrimary(value: unknown) {
&& !/I(?:'ve| have)? (?:set|made|changed).*(?:primary|go.to)/i.test(o.reply), "The model directs the personal action and does not falsely claim an agent preference write.");
return c.result();
}
export async function verifyReadyPrimaryChat(ctx: ChatCatalogContext, state: Record<string, unknown>, value: unknown) {
const o = runtimeObservation(value); const before = state.primaryBefore as Primary; const worker = state.worker as RuntimeAgent;
const sourceId = state.sourceBeforeNavigation; const nativeIds = state.nativeIdsBeforeNavigation;
if (!o || o.companyId !== ctx.input.fixtures.company.id || o.leadId !== ctx.input.fixtures.agent.id
|| typeof sourceId !== "string" || !sourceId || !Array.isArray(nativeIds) || nativeIds.length !== 1 || typeof nativeIds[0] !== "string")
throw new Error("Primary Chat readiness requires its exact original conversation and native run");
const budget = () => remainingIdentityObservationBudget(ctx);
const lead = o.agents.find(row => row.id === o.leadId); if (!lead) throw new Error("Primary Chat lead is missing");
await expect(ctx.input.page).toHaveURL(url => {
const parts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent);
return parts.length === 3 && parts[0] === ctx.input.fixtures.company.issuePrefix && parts[1] === "chats" && [lead.id, lead.urlKey].includes(parts[2]);
}, { timeout: budget() }); budget();
const composer = ctx.input.page.getByTestId("task-chat-composer-input");
await expect(composer).toBeVisible({ timeout: budget() }); budget();
await expect(composer.locator('[contenteditable="true"], textarea').first()).toBeEditable({ timeout: budget() }); budget();
const chat = await ctx.input.api.get<Entity & { conversationAgentId?: string; conversationUserId?: string }>(`/api/companies/${o.companyId}/chats/${o.leadId}`, { timeout: budget() }); budget();
if (chat.id !== sourceId || chat.companyId !== o.companyId || chat.conversationAgentId !== o.leadId || chat.conversationUserId !== before.userId)
throw new Error("Ready Chat reopened a different conversation or person");
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${sourceId}/comments?order=asc`, { timeout: budget() }); budget();
const humans = comments.filter(row => !row.authorAgentId);
const reply = comments.find(row => row.authorAgentId === o.leadId && row.createdByRunId === nativeIds[0] && !!row.body?.trim());
if (humans.length !== 1 || humans[0]!.authorUserId !== before.userId || humans[0]!.body?.trim() !== ctx.scenario.query.trim() || !reply)
throw new Error("Ready Chat lacks its original human request and native response");
for (const comment of [humans[0]!, reply]) {
await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${comment.id}`)}]`)).toBeVisible({ timeout: budget() }); budget();
}
await ctx.refreshIssue(); budget();
const primary = await ctx.input.api.get<Primary>(endpoint(o.companyId), { timeout: budget() }); budget();
const chats = await ctx.input.api.get<Entity[]>(`/api/companies/${o.companyId}/chats`, { timeout: budget() }); budget();
const runs = await ctx.allRuns(); budget();
if (ctx.issue()?.id !== sourceId || primary.companyId !== o.companyId || primary.userId !== before.userId || primary.primaryAgentId !== worker.id
|| chats.length !== state.chatCountBeforeNavigation || chats.filter(row => row.id === sourceId).length !== 1
|| runs.length !== 1 || runs[0]?.id !== nativeIds[0] || runs[0].agentId !== o.leadId || runs[0].status !== "succeeded")
throw new Error("Settled Chat changed the primary preference, conversation or native work");
state.reopenedRecentAgent = o.leadId; state.runsAfterNavigation = runs.length; state.chatCountAfterNavigation = chats.length;
await ctx.input.evidence("chat-primary-ready-conversation.json", { companyId: o.companyId, agentId: o.leadId, sourceId, userId: before.userId,
primaryAgentId: primary.primaryAgentId, originalNativeIds: nativeIds, chatCount: chats.length, composerReady: true, originalConversationVisible: true, noExtraWork: true }); budget();
}
export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", async ctx => {
const secrets = ctx.input.privateSecrets; if (!secrets) throw new Error("Primary fixture requires private credential sink");
const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Real browser required");
@@ -48,7 +86,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a
otherCompanyBefore: await ctx.input.api.get<Primary>(endpoint(otherCompany.id)) };
}, gradePrimary, {
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const worker = state.worker as RuntimeAgent;
state.beforeHumanPrimary = (await ctx.input.api.get<Primary>(endpoint(ctx.input.fixtures.company.id))).primaryAgentId;
@@ -67,12 +105,12 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
const chats = `/api/companies/${ctx.input.fixtures.company.id}/chats`;
state.chatCountBeforeNavigation = (await ctx.input.api.get<Entity[]>(chats)).length;
state.sourceBeforeNavigation = ctx.issue()?.id;
state.nativeIdsBeforeNavigation = (await ctx.allRuns()).map(run => run.id);
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/chats`, { waitUntil: "domcontentloaded" });
const lead = o.agents.find(row => row.id === ctx.input.fixtures.agent.id)!;
await ctx.input.page.waitForURL(url => [lead.id, lead.urlKey].includes(decodeURIComponent(url.pathname.split("/").at(-1)!)));
state.reopenedRecentAgent = ctx.input.fixtures.agent.id;
state.runsAfterNavigation = (await ctx.allRuns()).length;
state.chatCountAfterNavigation = (await ctx.input.api.get<Entity[]>(chats)).length;
await verifyReadyPrimaryChat(ctx, state, o);
},
async observe(ctx, state) {
const person = people.get(ctx); if (!person) throw new Error("Secondary authenticated person missing");
@@ -81,6 +119,12 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a
state.otherCompanyAfter = await ctx.input.api.get<Primary>(endpoint((state.otherCompany as Entity).id));
return observeRuntime(ctx, state);
},
async verifyResult(ctx, state, value) {
remainingIdentityObservationBudget(ctx);
await verifyRuntimeResult(ctx, state, value);
remainingIdentityObservationBudget(ctx);
await verifyReadyPrimaryChat(ctx, state, value);
},
async cleanup(ctx, state) {
const person = people.get(ctx); people.delete(ctx); await person?.dispose();
await ctx.input.api.delete(`/api/companies/${(state.otherCompany as Entity).id}`, { allowNotFound: true });