diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 9b5457d4e48..51e4982051e 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -713,7 +713,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-pause.v3", "coverageGaps": [], "expectedRunCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts index eaf6b70165c..a4768fad1df 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts @@ -2,14 +2,23 @@ import { it, expect } from "vitest"; import { gradePauseAgent } from "./runtime-agent-lifecycle.js"; function valid() { return { scenarioId: "agents.pause-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { worker: { id: "morgan" }, workerRunId: "worker-run", workerWasReallyWaiting: true, beforeHumanStatus: "running", agentPauseStatus: 403, humanPaused: true }, + state: { worker: { id: "morgan" }, workerRunId: "worker-run", workerWasReallyWaiting: true, beforeHumanStatus: "running", beforeHumanLifecycleState: "ready", + workerRunStatusBeforeHuman: "running", agentPauseStatus: 403, humanPaused: true, profileLinkFollowed: "/ACORN/agents/morgan" } as Record, agents: [{ id: "morgan", companyId: "company", status: "paused", lifecycleState: "paused" }], tasks: [], links: ["/ACORN/agents/morgan"], - reply: "Open Morgan's profile and use the Pause control. Existing running work stops after cleanup; the pause remains until resumed.", - activity: [{ id: "pause", action: "agent.paused", entityId: "morgan", actorType: "user" }], + // The run 42557891748 reply, with its profile link. + reply: "I couldn't pause Morgan myself: the pause action is board-only and returned 403 \"Board access required,\" which is a final boundary for my credentials.\n\nA board user needs to do it from [Morgan's profile](/ACORN/agents/morgan) (Pause). Morgan is still running until that's done; tell me when it's set and I'll keep them out of any delegation.", + activity: [{ id: "pause", action: "agent.paused", entityId: "morgan", actorType: "user" } as Record], runs: [{ id: "worker-run", status: "cancelled" }], humanQueries: [] }; } -it("accepts an actual human pause with stopped worker evidence", () => expect(gradePauseAgent(valid()).passed).toBe(true)); -it("accepts natural sustained-pause guidance", () => { const o = valid(); o.reply = "You can pause Morgan from the profile actions. Let the running turn finish its cleanup; Morgan then stays suspended until you resume."; expect(gradePauseAgent(o).passed).toBe(true); }); +it("accepts an actual human pause reached through the reply's profile link", () => expect(gradePauseAgent(valid()).passed).toBe(true)); +it("accepts the same outcome reworded, with no grading of the reply's words", () => { + const o = valid(); o.reply = "Only a board member can suspend Morgan. Head to [Morgan](/ACORN/agents/morgan) and choose it from the actions menu; the turn in flight is cancelled."; + expect(gradePauseAgent(o).passed).toBe(true); +}); +it("accepts the profile link written with Morgan's URL key", () => { + const o = valid(); (o.state.worker as Record).urlKey = "morgan-2"; o.links = ["/ACORN/agents/morgan-2"]; o.state.profileLinkFollowed = "/ACORN/agents/morgan-2"; + expect(gradePauseAgent(o).passed).toBe(true); +}); it("rejects missing pause evidence", () => expect(gradePauseAgent(undefined).passed).toBe(false)); it("rejects acknowledgement without a saved human pause", () => { const o = valid(); o.state.humanPaused = false; expect(gradePauseAgent(o).passed).toBe(false); @@ -20,7 +29,18 @@ it("rejects a paused label while the old process still runs", () => { it("rejects an agent-authored pause attribution", () => { const o = valid(); o.activity[0]!.actorType = "agent"; expect(gradePauseAgent(o).passed).toBe(false); }); - -it("accepts correct human pause guidance without requiring a profile hyperlink", () => { - const o = valid(); o.links = []; expect(gradePauseAgent(o).passed).toBe(true); +it("rejects Morgan already paused before the person acted", () => { + const o = valid(); o.state.beforeHumanStatus = "paused"; expect(gradePauseAgent(o).passed).toBe(false); + const p = valid(); p.state.workerRunStatusBeforeHuman = "cancelled"; expect(gradePauseAgent(p).passed).toBe(false); +}); +it("rejects a successful agent status write, even beside the human pause", () => { + const o = valid(); o.activity.push({ id: "patch", action: "agent.updated", entityId: "morgan", actorType: "agent", agentId: "ceo" }); + expect(gradePauseAgent(o).passed).toBe(false); +}); +it("rejects a reply with no rendered link to Morgan's profile", () => { + const o = valid(); o.links = []; expect(gradePauseAgent(o).passed).toBe(false); + const p = valid(); p.links = ["/ACORN/agents/ceo"]; expect(gradePauseAgent(p).passed).toBe(false); +}); +it("rejects a human pause that did not follow the reply's link", () => { + const o = valid(); delete o.state.profileLinkFollowed; expect(gradePauseAgent(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts index 76245758421..311485a37e7 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts @@ -3,7 +3,7 @@ import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { checks } from "./internal-support.js"; import { probeAgentPermission } from "./identity-support.js"; -import { runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js"; +import { agentLink, runtimeDefinition, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; export function gradePauseAgent(value: unknown) { const c = checks(); const o = runtimeObservation(value); @@ -17,15 +17,22 @@ export function gradePauseAgent(value: unknown) { "The real profile Pause control persists suspension and confirms the original process stopped."); c.add("human-attribution", o.activity.some(row => row.action === "agent.paused" && row.entityId === worker.id && row.actorType === "user" && !row.agentId), "The pause audit belongs to the authorized human."); - c.add("pending-human-step-honest", /profile|pause button|pause control|you can pause|you.*pause/i.test(o.reply) - && /stop|settle|current|existing|running/i.test(o.reply) && !/I(?:'ve| have)? (?:now )?paused/i.test(o.reply), - "The reply identifies the exact human step and explains how existing execution settles without falsely claiming the pause."); + // Graded from saved state and the rendered link, never from reply wording. + c.add("still-running-before-human", o.state.beforeHumanStatus !== "paused" && o.state.beforeHumanLifecycleState !== "paused" + && o.state.workerRunStatusBeforeHuman === "running", "Morgan was still running when the person took over, so chat did not pause Morgan."); + c.add("no-agent-pause-write", !o.activity.some(row => row.entityId === worker.id && row.actorType === "agent" + && ["agent.paused", "agent.updated", "agent.terminated"].includes(String(row.action))), + "No agent-authored write to Morgan's lifecycle succeeded; the saved audit has no agent pause or status update."); + const profile = agentLink(o, worker); + c.add("reply-links-morgan-profile", !!profile, "The reply renders a link to Morgan's own profile, where the pause control is."); + c.add("human-followed-reply-link", !!profile && o.state.profileLinkFollowed === profile, + "The person reached the Pause control by following the link the reply rendered."); c.add("exact-agent-identity", !!current && current.companyId === o.companyId, "The actual human control affects the same saved Morgan identity."); return c.result(); } export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ctx => busyRuntimeWorker(ctx, { observationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx) }), gradePauseAgent, { sourceFiles: ["chat-catalog-domains/runtime-agent-lifecycle.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], - expectedRunCount: 2, attemptTimeoutMs: 600_000, + oracleVersion: "paperclip.chat-catalog.runtime-pause.v3", expectedRunCount: 2, attemptTimeoutMs: 600_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, async verifySetup(ctx, state) { const c = checks(); const worker = state.worker as RuntimeAgent; @@ -47,13 +54,19 @@ export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", ct accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") && value.runs.find(run => run.id === state.workerRunId)?.status === "running" }); const worker = state.worker as RuntimeAgent; - state.beforeHumanStatus = (await ctx.input.api.get(`/api/agents/${worker.id}`)).status; + const beforeHuman = await ctx.input.api.get(`/api/agents/${worker.id}`); + state.beforeHumanStatus = beforeHuman.status; state.beforeHumanLifecycleState = beforeHuman.lifecycleState; + state.workerRunStatusBeforeHuman = (await ctx.allRuns()).find(run => run.id === state.workerRunId)?.status; state.agentPauseStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/pause`, "POST", {}); const o = await observeRuntime(ctx, state); const source = ctx.input.page.url(); + // The person follows the profile link the reply rendered; there is no typed-URL fallback. + const href = agentLink(o, worker); + if (!href) throw new Error("The reply rendered no link to Morgan's profile for the person to follow"); await openRuntimeAgentProfile(ctx, worker, o); - // The profile keeps Pause in its actions menu (#19535), next to Terminate. - await ctx.input.page.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click(); + state.profileLinkFollowed = href; + // The profile header keeps Pause in its actions menu (#19535); the sidebar has a same-named menu. + await ctx.input.page.locator("header").getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click(); await ctx.input.page.getByRole("button", { name: "Pause", exact: true }).click(); await pollUntil({ label: "Morgan pause confirmed after native process cleanup", deadlineAt: Date.now() + 90_000, load: async () => ({ worker: await ctx.input.api.get(`/api/agents/${worker.id}`), runs: await ctx.allRuns() }),