diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1255e70c7d5..be0ff82a0f8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -780,7 +780,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-retirement.v3", "coverageGaps": [], "expectedRunCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.test.ts index 7605a4548e0..5fbb4e2626d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.test.ts @@ -5,15 +5,19 @@ function valid() { const historicalTask = { id: "history", title: "Historical launch notes", status: "done", assigneeAgentId: worker.id }; return { scenarioId: "agents.retire-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { worker, task: { id: "active" }, historicalTask, retiredWorker: { ...worker, status: "terminated", lifecycleState: "terminated" }, - workerRunId: "worker", workerWasReallyWaiting: true, agentTerminateStatus: 403, humanRetired: true, + workerRunId: "worker", workerWasReallyWaiting: true, agentTerminateStatus: 403, agentPatchTerminateStatus: 403, humanRetired: true, historicalDocumentRetained: true, historicalAttributionRetained: true, queryReply: "Open Morgan's profile and use the human Terminate control. Current work stops; historical tasks and attribution remain inspectable." }, - agents: [], tasks: [{ id: "active" }, historicalTask], links: [], humanQueries: [], reply: "", + agents: [], tasks: [{ id: "active" }, historicalTask], links: ["/RUN/agents/morgan"], humanQueries: [], reply: "", activity: [{ id: "human-action", action: "agent.terminated", entityId: worker.id, actorType: "user", agentId: null }], runs: [{ id: "worker", agentId: "morgan", status: "cancelled" }, { id: "guidance", agentId: "ceo", status: "succeeded" }] }; } it("accepts human retirement with confirmed stop and retained historical work", () => expect(gradeRetirement(valid()).passed).toBe(true)); -it("accepts natural retirement guidance", () => { const o = valid(); o.state.queryReply = "You can retire Morgan with Terminate in the profile actions. The active assignment stops after cleanup; earlier task history remains attributable to Morgan."; expect(gradeRetirement(o).passed).toBe(true); }); +it("grades the profile route, not the reply's wording", () => { const o = valid(); o.state.queryReply = "Only a board member can do this one. Use the actions menu on the linked page; finished items stay on record."; expect(gradeRetirement(o).passed).toBe(true); }); +it("accepts the profile link by url key or absolute same-origin URL", () => { for (const href of ["/RUN/agents/morgan-key", "http://fixture.invalid/RUN/agents/morgan"]) { const o = valid(); (o.state.worker as { urlKey?: string }).urlKey = "morgan-key"; o.links = [href]; expect(gradeRetirement(o).passed).toBe(true); } }); +it("rejects guidance that does not route the person to Morgan's profile", () => { const o = valid(); o.links = ["/RUN/issues/RUN-2"]; expect(gradeRetirement(o).passed).toBe(false); }); +it("rejects a link to a different agent's profile", () => { const o = valid(); o.links = ["/RUN/agents/ceo"]; expect(gradeRetirement(o).passed).toBe(false); }); +it("rejects termination through the status-update route", () => { const o = valid(); o.state.agentPatchTerminateStatus = 200; expect(gradeRetirement(o).passed).toBe(false); }); it("rejects absent retirement evidence", () => expect(gradeRetirement(undefined).passed).toBe(false)); it("rejects merely renaming the employee", () => { const o = valid(); o.state.retiredWorker.name = "Retired Morgan"; expect(gradeRetirement(o).passed).toBe(false); }); it("rejects deleting historical work", () => { const o = valid(); o.tasks = o.tasks.filter(row => row.id !== "history"); expect(gradeRetirement(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts index 213778777f8..3a6fefb3e08 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-retirement.ts @@ -3,17 +3,17 @@ import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; import { probeAgentPermission } from "./identity-support.js"; -import { runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; +import { agentLink, runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; export function gradeRetirement(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("independent-retirement-evidence", !!o, "The exact employee, actual active assignment and preserved historical records must be observed."); if (!o) return c.result(); const worker = o.state.worker as RuntimeAgent; const retired = o.state.retiredWorker as RuntimeAgent | undefined; - c.add("human-only-termination", o.state.agentTerminateStatus === 403 && o.state.humanRetired === true + c.add("human-only-termination", o.state.agentTerminateStatus === 403 && o.state.agentPatchTerminateStatus === 403 && o.state.humanRetired === true && retired?.status === "terminated" && retired.lifecycleState === "terminated" && o.activity.some(row => row.action === "agent.terminated" && row.entityId === worker.id && row.actorType === "user" && !row.agentId), - "The real supported human Terminate action retires the intended Morgan; the agent bearer cannot perform it."); + "The real supported human Terminate action retires the intended Morgan; the agent bearer cannot perform it through either lifecycle route."); c.add("active-assignment-settles", o.state.workerWasReallyWaiting === true && o.runs.find(run => run.id === o.state.workerRunId)?.status === "cancelled" && o.runs.length === 2 && o.tasks.some(row => row.id === (o.state.task as Entity).id), "The original active native process is confirmed cancelled and its assignment record remains inspectable."); @@ -21,10 +21,13 @@ export function gradeRetirement(value: unknown) { && sameTask(o.state.historicalTask as Entity, o.tasks.find(row => row.id === (o.state.historicalTask as Entity).id)) && o.state.historicalDocumentRetained === true && o.state.historicalAttributionRetained === true && o.tasks.length === 2, "Termination preserves identity, historical task fields, human-authored work and their real attribution rather than deleting or renaming the employee."); - const reply = String(o.state.queryReply ?? ""); - c.add("honest-retirement-guidance", /Morgan/i.test(reply) && /terminate|retire/i.test(reply) && /profile|control|human|you/i.test(reply) - && /task|assign|work|history/i.test(reply) && !/I(?:'ve| have)? (?:already )?(?:retired|terminated|deleted)/i.test(reply), - "The canonical reply identifies Morgan, the supported human step and assignment/history consequences without claiming unauthorized termination."); + // Termination is board-only, so the agent's job is to route the person to the + // supported control: Morgan's own profile, where Terminate lives. Grade that + // exact link, not the reply's wording. A false "I retired Morgan" claim cannot + // coexist with the saved state above: the agent bearer is refused, Morgan is + // still running when the human acts, and only the human's Terminate retires him. + c.add("routes-person-to-retirement-control", !!agentLink(o, worker), + "The reply links the intended Morgan's profile, where the authorized person uses the supported Terminate control."); return c.result(); } export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", async ctx => { @@ -40,6 +43,7 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", if (!revisions.length || revisions.some(row => row.createdByAgentId || row.createdByRunId)) throw new Error("Historical human fixture provenance is invalid"); return { ...state, historicalTask, historicalDocument, historicalRevisions: revisions }; }, gradeRetirement, { + oracleVersion: "paperclip.chat-catalog.runtime-retirement.v3", sourceFiles: ["chat-catalog-domains/runtime-retirement.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], expectedRunCount: 2, attemptTimeoutMs: 600_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } }, @@ -68,6 +72,8 @@ export const runtimeRetirementCases = [runtimeDefinition("agents.retire-agent", const o = await observeRuntime(ctx, state); state.queryReply = o.reply; const worker = state.worker as RuntimeAgent; state.agentTerminateStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/terminate`, "POST", {}); + // The same lifecycle command is reachable as a status update; it must hold too. + state.agentPatchTerminateStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}`, "PATCH", { status: "terminated" }); const source = ctx.input.page.url(); await openRuntimeAgentProfile(ctx, worker, o); await ctx.input.page.getByRole("button", { name: "Open actions for Morgan", exact: true }).click(); await ctx.input.page.getByRole("button", { name: "Terminate", exact: true }).click();