mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
test(e2e): cover runtime history and human primary controls
This commit is contained in:
1 parent
b5f0e6399f
commit
45b248d2f0
7 files changed
+288
-6
No files matched your search
@@ -0,0 +1,19 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeReopenConversation } from "./runtime-agent-chat.js";
|
||||
function valid() {
|
||||
return { scenarioId: "agent-chat.reopen-conversation", companyId: "company", leadId: "morgan", reply: "Your existing conversation.",
|
||||
state: { existingIssueId: "chat", reopenedIssueId: "chat", reopenedAgentId: "morgan", historyRetained: true,
|
||||
beforeNavigationRuns: 2, afterNavigationRuns: 2, beforeNavigationChatIds: ["chat"], afterNavigationChatIds: ["chat"],
|
||||
agentChatListStatus: 403, linkMatchedFinalRun: true }, agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [] };
|
||||
}
|
||||
it("accepts original conversation navigation retaining genuine history", () => expect(gradeReopenConversation(valid()).passed).toBe(true));
|
||||
it("rejects missing conversation evidence", () => expect(gradeReopenConversation(undefined).passed).toBe(false));
|
||||
it("rejects opening a duplicate conversation", () => {
|
||||
const o = valid(); o.state.reopenedIssueId = "duplicate"; expect(gradeReopenConversation(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects navigation that starts provider work", () => {
|
||||
const o = valid(); o.state.afterNavigationRuns = 3; expect(gradeReopenConversation(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects a link from the setup reply instead of canonical query", () => {
|
||||
const o = valid(); o.state.linkMatchedFinalRun = false; expect(gradeReopenConversation(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { sendChatMessage } from "../chat-flow.js";
|
||||
import { checks, type Entity } from "./internal-support.js";
|
||||
import { readAgentResource, openStandingChat } from "./identity-support.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
|
||||
|
||||
export function gradeReopenConversation(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("persisted-reopen-evidence", !!o, "The existing conversation, original attributed history and navigation result are independently read."); if (!o) return c.result();
|
||||
c.add("same-existing-owned-conversation", typeof o.state.existingIssueId === "string" && o.state.reopenedIssueId === o.state.existingIssueId
|
||||
&& o.state.reopenedAgentId === o.leadId && o.state.historyRetained === true,
|
||||
"The supported result opens the same person's existing Morgan conversation and retains its real prior turn.");
|
||||
c.add("navigation-started-no-work", o.state.beforeNavigationRuns === 2 && o.state.afterNavigationRuns === 2
|
||||
&& o.state.beforeNavigationChatIds !== undefined && JSON.stringify(o.state.beforeNavigationChatIds) === JSON.stringify(o.state.afterNavigationChatIds)
|
||||
&& o.tasks.length === 0,
|
||||
"The initial setup and canonical query are the only two runs; navigation creates neither work nor a duplicate standing chat.");
|
||||
c.add("private-chat-list-not-readable-to-agent", o.state.agentChatListStatus === 403,
|
||||
"The actual agent bearer cannot enumerate the person's private conversation list.");
|
||||
c.add("canonical-query-response-link", o.state.linkMatchedFinalRun === true,
|
||||
"A link to the actual existing conversation was rendered in a reply attributed to the canonical query's run.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeAgentChatCases = [runtimeDefinition("agent-chat.reopen-conversation", async ctx => {
|
||||
await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}`, { name: "Morgan" });
|
||||
const source = await openStandingChat(ctx);
|
||||
const lead = await ctx.input.api.get<Entity>(`/api/agents/${ctx.input.fixtures.agent.id}`);
|
||||
const historyMarker = `Retained launch context ${ctx.input.nonce}`;
|
||||
await sendChatMessage(ctx.input.page, `Remember this launch reference in our conversation: ${historyMarker}. Acknowledge briefly; no task is needed.`);
|
||||
await pollUntil({ label: "Genuine prior Morgan conversation", deadlineAt: Date.now() + 150_000,
|
||||
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: ({ source, runs }) => source?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded") });
|
||||
const history = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
|
||||
const firstRuns = await ctx.allRuns();
|
||||
if (!history.some(row => row.authorAgentId === ctx.input.fixtures.agent.id && firstRuns.some(run => run.id === row.createdByRunId)))
|
||||
throw new Error("Existing Morgan history requires a genuine attributed native reply");
|
||||
const read = await readAgentResource(ctx, `/api/companies/${ctx.input.fixtures.company.id}/chats`);
|
||||
return { existingIssueId: source.id, lead, history, historyMarker, setupRunIds: firstRuns.map(run => run.id), agentChatListStatus: read.status };
|
||||
}, gradeReopenConversation, {
|
||||
authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 420_000,
|
||||
async act(ctx, state) {
|
||||
await pollUntil({ label: "Canonical reopen query settled", deadlineAt: Date.now() + 150_000,
|
||||
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: ({ source, runs }) => source?.conversationState === "waiting" && runs.length === 2 && runs.every(run => run.status === "succeeded") });
|
||||
const source = ctx.issue()!; const runs = await ctx.allRuns();
|
||||
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
|
||||
const setupIds = state.setupRunIds as string[];
|
||||
const finalReplies = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id
|
||||
&& runs.some(run => run.id === row.createdByRunId && !setupIds.includes(run.id)));
|
||||
let href: string | undefined;
|
||||
for (const reply of finalReplies) {
|
||||
const links = await ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}] a[href]`).evaluateAll(rows => rows.map(row => row.getAttribute("href")!));
|
||||
href = links.find(link => {
|
||||
const url = new URL(link, ctx.input.api.baseURL);
|
||||
return url.origin === ctx.input.api.baseURL && url.pathname.split("/").includes("chats")
|
||||
&& [ctx.input.fixtures.agent.id, (state.lead as Entity).urlKey].includes(decodeURIComponent(url.pathname.split("/").at(-1)!));
|
||||
});
|
||||
if (href) break;
|
||||
}
|
||||
if (!href) throw new Error("Canonical reopen response lacks the existing Morgan conversation link");
|
||||
state.linkMatchedFinalRun = true; state.beforeNavigationRuns = runs.length;
|
||||
const chatList = `/api/companies/${ctx.input.fixtures.company.id}/chats`;
|
||||
state.beforeNavigationChatIds = (await ctx.input.api.get<Entity[]>(chatList)).map(row => row.id).sort();
|
||||
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).last().click();
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
await ctx.refreshIssue(); state.reopenedIssueId = ctx.issue()?.id;
|
||||
const reopened = await ctx.input.api.get<{ conversationAgentId: string }>(`/api/issues/${ctx.issue()!.id}`);
|
||||
state.reopenedAgentId = reopened.conversationAgentId;
|
||||
const after = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
|
||||
state.historyRetained = (state.history as Entity[]).every(prior => after.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId));
|
||||
state.afterNavigationRuns = (await ctx.allRuns()).length;
|
||||
state.afterNavigationChatIds = (await ctx.input.api.get<Entity[]>(chatList)).map(row => row.id).sort();
|
||||
},
|
||||
observe: observeRuntime,
|
||||
async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
|
||||
})];
|
||||
@@ -17,6 +17,17 @@ function valid(id: string): RuntimeObservation {
|
||||
o.agents = [o.agents[0]!]; delete o.state.worker;
|
||||
Object.assign(o.state, { canCreateAgents: false, agentCreateProbeStatus: 403 });
|
||||
o.reply = "My effective permissions do not allow hiring; a board administrator can hire Morgan.";
|
||||
} else if (id === "agents.edit-instructions") {
|
||||
const revision = { id: "original", agentId: "morgan", companyId: "company", source: "seed" };
|
||||
o.state.originalInstruction = { content: "Original frontend guidance", revision };
|
||||
o.state.concurrentMarker = "Keep keyboard support";
|
||||
o.state.concurrentInstruction = { content: "Original frontend guidance\nKeep keyboard support", revision: { ...revision, id: "concurrent", source: "board" } };
|
||||
o.state.concurrentRunStatus = "running";
|
||||
o.state.sideContent = "Keep accessibility"; o.instructionSideFile = { content: "Keep accessibility" };
|
||||
o.instruction = { path: "AGENTS.md", content: "Original frontend guidance\nKeep keyboard support\nRun the relevant tests before handing work back.",
|
||||
revision: { ...revision, id: "merged", source: "api", actorAgentId: "ceo", sourceRunId: "run" } };
|
||||
o.instructionHistory = { revisions: [revision, { ...revision, id: "concurrent" }, o.instruction.revision] };
|
||||
o.reply = "Added the rule to Morgan's instructions for future work.";
|
||||
} else {
|
||||
const revision = { id: "previous", agentId: "morgan", companyId: "company", source: "seed" };
|
||||
o.state.previousInstruction = { path: "AGENTS.md", content: "Original guidance", revision };
|
||||
@@ -65,3 +76,18 @@ describe("runtime agent oracles", () => {
|
||||
expect(gradeRuntimeAgents(o).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an instruction edit that overwrote concurrent human guidance", () => {
|
||||
const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nRun the relevant tests before handing work back.";
|
||||
expect(gradeRuntimeAgents(o).passed).toBe(false);
|
||||
});
|
||||
it("accepts an explicit unresolved revision conflict preserving the concurrent edit", () => {
|
||||
const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support";
|
||||
o.reply = "Morgan's instruction revision changed concurrently; the testing-rule update has an unresolved conflict.";
|
||||
expect(gradeRuntimeAgents(o).passed).toBe(true);
|
||||
});
|
||||
it("rejects claimed success after a conflict left the rule unsaved", () => {
|
||||
const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support";
|
||||
o.reply = "There was a revision conflict, but I have added the testing rule for future work.";
|
||||
expect(gradeRuntimeAgents(o).passed).toBe(false);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { checks, sameTask } from "./internal-support.js";
|
||||
import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, agentLink, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
|
||||
export function gradeRuntimeAgents(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
@@ -30,6 +31,26 @@ export function gradeRuntimeAgents(value: unknown) {
|
||||
&& /board|human|owner|administrator|admin/i.test(o.reply) && !/missing.*(?:Anthropic|account)|connect.*Anthropic/i.test(o.reply),
|
||||
"The response identifies effective permission and an authorized human route rather than blaming account setup.");
|
||||
break;
|
||||
case "agents.edit-instructions": {
|
||||
const original = o.state.originalInstruction as InstructionFile | undefined;
|
||||
const concurrent = o.state.concurrentInstruction as InstructionFile | undefined;
|
||||
const current = o.instruction;
|
||||
const ruleSaved = !!current && /(?:run|execute)[^\n.!]*?(?:relevant|appropriate|applicable)[^\n.!]*?tests?[^\n.!]*?(?:hand|return|finish|complet)|(?:before|prior to)[^\n.!]*?(?:hand|return|finish|complet)[^\n.!]*?(?:run|execut)[^\n.!]*?tests?/i.test(current.content);
|
||||
c.add("concurrent-guidance-preserved", !!original && !!concurrent && !!current && current.content.includes(original.content)
|
||||
&& current.content.includes(String(o.state.concurrentMarker)) && o.instructionSideFile?.content === o.state.sideContent,
|
||||
"The saved entry retains prior and concurrently authored human guidance, and the unrelated file is unchanged.");
|
||||
c.add("requested-rule-or-visible-conflict", ruleSaved || /conflict|changed while|newer revision|concurrent|stale/i.test(o.reply)
|
||||
&& !/(?:I(?:'ve| have)? (?:added|saved)|successfully (?:added|saved))[^.!]*?(?:rule|instruction|test)/i.test(o.reply),
|
||||
"The rule is durably merged or the unresolved revision conflict is visibly reported without false success.");
|
||||
c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!concurrent
|
||||
&& o.instructionHistory?.revisions.some(row => row.id === concurrent.revision.id) === true,
|
||||
"An actual human revision occurred during the canonical query's active native run and remains in history.");
|
||||
c.add("rule-attributed-if-saved", !ruleSaved || !!current && current.revision.actorAgentId === o.leadId
|
||||
&& o.runs.some(run => run.id === current.revision.sourceRunId), "A saved testing rule belongs to this actual agent run.");
|
||||
c.add("future-use-and-profile-link", !!worker && !!agentLink(o, worker) && /future|next|subsequent|new (?:run|work)|conflict|revision/i.test(o.reply),
|
||||
"The reply links Morgan and describes future instruction use or the unresolved revision.");
|
||||
break;
|
||||
}
|
||||
case "agents.restore-instructions": {
|
||||
const previous = o.state.previousInstruction as InstructionFile; const unwanted = o.state.unwantedInstruction as InstructionFile;
|
||||
const current = o.instruction;
|
||||
@@ -50,6 +71,27 @@ export function gradeRuntimeAgents(value: unknown) {
|
||||
}
|
||||
|
||||
export const runtimeAgentsCases = [
|
||||
runtimeDefinition("agents.edit-instructions", async ctx => {
|
||||
const sideContent = `Retain accessibility guidance ${ctx.input.nonce}.`;
|
||||
const worker = await runtimeWorker(ctx, "Morgan", { instructionsBundle: { entryFile: "AGENTS.md", files: {
|
||||
"AGENTS.md": `Preserve the original frontend guidance ${ctx.input.nonce}.`, "guidance.md": sideContent,
|
||||
} } });
|
||||
const originalInstruction = await ctx.input.api.get<InstructionFile>(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`);
|
||||
return { worker, instructionAgentId: worker.id, originalInstruction, sideContent, concurrentMarker: `Concurrent human guidance ${ctx.input.nonce}: preserve keyboard support.` };
|
||||
}, gradeRuntimeAgents, {
|
||||
async act(ctx, state) {
|
||||
const active = await pollUntil({ label: "Active instructions update before concurrent human revision", deadlineAt: Date.now() + 60_000,
|
||||
load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running",
|
||||
reject: runs => runs.some(run => ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status)) ? "Instruction run settled before concurrent fixture boundary" : undefined });
|
||||
state.concurrentRunStatus = active[0]!.status;
|
||||
const root = `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle/file`;
|
||||
const current = await ctx.input.api.get<InstructionFile>(`${root}?path=AGENTS.md`);
|
||||
if (current.revision.id !== (state.originalInstruction as InstructionFile).revision.id)
|
||||
throw new Error("Agent already changed the entry before the concurrent fixture boundary");
|
||||
state.concurrentInstruction = await ctx.input.api.putSensitive<InstructionFile>(root, { path: "AGENTS.md",
|
||||
content: `${current.content}\n${state.concurrentMarker}`, baseRevisionId: current.revision.id });
|
||||
},
|
||||
}),
|
||||
runtimeDefinition("agents.change-manager", async ctx => {
|
||||
const worker = await runtimeWorker(ctx); const riley = await runtimeWorker(ctx, "Riley", { role: "cto", title: "CTO" });
|
||||
const task = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Historical website assignment", status: "backlog", assigneeAgentId: worker.id });
|
||||
|
||||
@@ -31,8 +31,8 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "agents.edit-instructions",
|
||||
"readinessClass": "local",
|
||||
"needs": "Versioned file with unrelated guidance; agent edit plus genuine concurrent revision conflict/fresh merge branch."
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Versioned instructions with genuine concurrent public human revision during active query; preserve unrelated guidance, merge testing rule or expose conflict."
|
||||
},
|
||||
{
|
||||
"scenarioId": "agents.restore-instructions",
|
||||
@@ -46,8 +46,8 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "agents.choose-primary",
|
||||
"readinessClass": "local",
|
||||
"needs": "Authenticated human preference and existing primary; profile replacement confirmation, other user/company scope and recent-chat precedence."
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Real authenticated primary replacement, visible confirmation, independently unchanged secondary person/company and recent-chat precedence without new runs."
|
||||
},
|
||||
{
|
||||
"scenarioId": "agents.pause-agent",
|
||||
@@ -86,8 +86,8 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "agent-chat.reopen-conversation",
|
||||
"readinessClass": "source",
|
||||
"needs": "Existing human-owned Morgan conversation and history; current shared source fixed to lead chat, cross-agent navigation must not duplicate or run."
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Genuine native setup history and declared canonical query boundary; same owned Morgan source, actual final-run result link, private-list denial and no navigation execution."
|
||||
},
|
||||
{
|
||||
"scenarioId": "agent-chat.remember-and-follow-up",
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradePrimary } from "./runtime-primary.js";
|
||||
function valid() {
|
||||
const primaryBefore = { companyId: "company", userId: "person", primaryAgentId: "ceo" };
|
||||
const other = { companyId: "company", userId: "other-person", primaryAgentId: null as string | null };
|
||||
return { scenarioId: "agents.choose-primary", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
|
||||
state: { worker: { id: "morgan" }, primaryBefore, primaryAfter: { ...primaryBefore, primaryAgentId: "morgan" }, beforeHumanPrimary: "ceo",
|
||||
replacementConfirmed: true, agentWriteStatus: 403, otherPersonBefore: other, otherPersonAfter: { ...other },
|
||||
otherCompanyBefore: { companyId: "other-company", primaryAgentId: null }, otherCompanyAfter: { companyId: "other-company", primaryAgentId: null },
|
||||
reopenedRecentAgent: "ceo", runsAfterNavigation: 1, chatCountBeforeNavigation: 1, chatCountAfterNavigation: 1 },
|
||||
agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [],
|
||||
reply: "Open Morgan's profile and choose Set as my primary; confirm replacing your personal primary." };
|
||||
}
|
||||
it("accepts a confirmed personal preference with isolated scope", () => expect(gradePrimary(valid()).passed).toBe(true));
|
||||
it("rejects missing primary evidence", () => expect(gradePrimary(undefined).passed).toBe(false));
|
||||
it("rejects an agent preference write before the human control", () => {
|
||||
const o = valid(); o.state.beforeHumanPrimary = "morgan"; expect(gradePrimary(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects a preference changed for another person", () => {
|
||||
const o = valid(); o.state.otherPersonAfter = { ...o.state.otherPersonAfter, primaryAgentId: "morgan" };
|
||||
expect(gradePrimary(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects navigation that starts a new execution", () => {
|
||||
const o = valid(); o.state.runsAfterNavigation = 2; expect(gradePrimary(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects false agent-authored completion language", () => {
|
||||
const o = valid(); o.reply = "I have made Morgan your primary. Open the profile to confirm.";
|
||||
expect(gradePrimary(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { checks, type Entity } from "./internal-support.js";
|
||||
import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
|
||||
import { probeAgentPermission, waitForReply } from "./identity-support.js";
|
||||
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent } from "./runtime-support.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
interface Primary { companyId: string; userId: string; primaryAgentId: string | null }
|
||||
type Person = Awaited<ReturnType<typeof createAuthenticatedPerson>>;
|
||||
const people = new WeakMap<ChatCatalogContext, Person>();
|
||||
const endpoint = (companyId: string) => `/api/companies/${companyId}/primary-agent/me`;
|
||||
async function personPrimary(person: Person, companyId: string): Promise<Primary> {
|
||||
const response = await person.request(endpoint(companyId));
|
||||
if (!response.ok) { await response.body?.cancel(); throw new Error(`Secondary primary read returned ${response.status}`); }
|
||||
return response.json() as Promise<Primary>;
|
||||
}
|
||||
export function gradePrimary(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("complete-human-primary-evidence", !!o, "Real user-scoped saved preferences and human replacement control are required."); if (!o) return c.result();
|
||||
const worker = o.state.worker as RuntimeAgent; const before = o.state.primaryBefore as Primary;
|
||||
const after = o.state.primaryAfter as Primary | undefined; const otherBefore = o.state.otherPersonBefore as Primary;
|
||||
const otherAfter = o.state.otherPersonAfter as Primary | undefined;
|
||||
c.add("only-human-replaces-existing-primary", !!before && before.primaryAgentId === o.leadId
|
||||
&& o.state.beforeHumanPrimary === o.leadId && o.state.replacementConfirmed === true
|
||||
&& after?.primaryAgentId === worker.id && after.companyId === o.companyId && after.userId === before.userId,
|
||||
"The original primary survives the chat response and is replaced only after the visible human confirmation.");
|
||||
c.add("agent-preference-write-denied", o.state.agentWriteStatus === 403, "The preference endpoint rejects the real CEO agent bearer.");
|
||||
c.add("other-person-company-unaffected", !!otherAfter && !!otherBefore && otherAfter.userId !== before.userId
|
||||
&& JSON.stringify(otherAfter) === JSON.stringify(otherBefore)
|
||||
&& JSON.stringify(o.state.otherCompanyAfter) === JSON.stringify(o.state.otherCompanyBefore),
|
||||
"Independent authenticated person and company preference readbacks do not change.");
|
||||
c.add("recent-chat-precedence-without-work", o.state.reopenedRecentAgent === o.leadId
|
||||
&& o.state.runsAfterNavigation === 1 && o.state.chatCountBeforeNavigation === o.state.chatCountAfterNavigation,
|
||||
"Opening Chat may reopen this valid recent source conversation and creates no duplicate chat or execution.");
|
||||
c.add("honest-human-control-guidance", /primary|go.to/i.test(o.reply) && /confirm|profile|you can|choose|set as my primary/i.test(o.reply)
|
||||
&& !/I(?:'ve| have)? (?:set|made|changed).*(?:primary|go.to)/i.test(o.reply), "The model directs the personal action and does not falsely claim an agent preference write.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", async ctx => {
|
||||
const secrets = ctx.input.privateSecrets; if (!secrets) throw new Error("Primary fixture requires private credential sink");
|
||||
const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Real browser required");
|
||||
const primaryBefore = await ctx.input.api.get<Primary>(endpoint(ctx.input.fixtures.company.id));
|
||||
if (primaryBefore.primaryAgentId !== ctx.input.fixtures.agent.id) throw new Error("Catalog requires an existing primary agent");
|
||||
const worker = await runtimeWorker(ctx);
|
||||
const person = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets, name: "Second primary reader" }); people.set(ctx, person);
|
||||
await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, person, secrets);
|
||||
const otherCompany = await ctx.input.api.post<Entity>("/api/companies", { name: `Other primary company ${ctx.input.nonce}` });
|
||||
return { worker, primaryBefore, otherCompany, otherPersonBefore: await personPrimary(person, ctx.input.fixtures.company.id),
|
||||
otherCompanyBefore: await ctx.input.api.get<Primary>(endpoint(otherCompany.id)) };
|
||||
}, gradePrimary, {
|
||||
authentication: "human-session",
|
||||
sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const worker = state.worker as RuntimeAgent;
|
||||
state.beforeHumanPrimary = (await ctx.input.api.get<Primary>(endpoint(ctx.input.fixtures.company.id))).primaryAgentId;
|
||||
state.agentWriteStatus = await probeAgentPermission(ctx, endpoint(ctx.input.fixtures.company.id), "PUT", { primaryAgentId: worker.id });
|
||||
const o = await observeRuntime(ctx, state); const href = agentLink(o, worker); if (!href) throw new Error("Primary guidance lacks Morgan's real profile link");
|
||||
const source = ctx.input.page.url();
|
||||
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
await ctx.input.page.getByRole("button", { name: "Set as my primary", exact: true }).click();
|
||||
const dialog = ctx.input.page.getByRole("alertdialog");
|
||||
await expect(dialog).toContainText("Change your primary agent?"); await expect(dialog).toContainText("Morgan");
|
||||
await dialog.getByRole("button", { name: "Set as my primary", exact: true }).click();
|
||||
await expect(ctx.input.page.getByRole("img", { name: "My primary agent", exact: true }).first()).toBeVisible();
|
||||
state.replacementConfirmed = true;
|
||||
// Revisit the valid recent source before opening the normal Chat landing.
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
const chats = `/api/companies/${ctx.input.fixtures.company.id}/chats`;
|
||||
state.chatCountBeforeNavigation = (await ctx.input.api.get<Entity[]>(chats)).length;
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/chats`, { waitUntil: "domcontentloaded" });
|
||||
const lead = o.agents.find(row => row.id === ctx.input.fixtures.agent.id)!;
|
||||
await ctx.input.page.waitForURL(url => [lead.id, lead.urlKey].includes(decodeURIComponent(url.pathname.split("/").at(-1)!)));
|
||||
state.reopenedRecentAgent = ctx.input.fixtures.agent.id;
|
||||
state.runsAfterNavigation = (await ctx.allRuns()).length;
|
||||
state.chatCountAfterNavigation = (await ctx.input.api.get<Entity[]>(chats)).length;
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const person = people.get(ctx); if (!person) throw new Error("Secondary authenticated person missing");
|
||||
state.primaryAfter = await ctx.input.api.get<Primary>(endpoint(ctx.input.fixtures.company.id));
|
||||
state.otherPersonAfter = await personPrimary(person, ctx.input.fixtures.company.id);
|
||||
state.otherCompanyAfter = await ctx.input.api.get<Primary>(endpoint((state.otherCompany as Entity).id));
|
||||
return observeRuntime(ctx, state);
|
||||
},
|
||||
async cleanup(ctx, state) {
|
||||
const person = people.get(ctx); people.delete(ctx); await person?.dispose();
|
||||
await ctx.input.api.delete(`/api/companies/${(state.otherCompany as Entity).id}`, { allowNotFound: true });
|
||||
},
|
||||
})];
|
||||
Reference in new issue
Block a user