diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 356416d7c77..36f8fe4d143 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -756,7 +756,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-resume.v3", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts index 3b70af35d45..0a0615c1f82 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts @@ -1,13 +1,13 @@ import { it, expect } from "vitest"; import { gradeResumeAgent } from "./runtime-resume.js"; -function valid() { +function valid(): any { const worker = { id: "morgan", companyId: "company" }; return { scenarioId: "agents.resume-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { worker, firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] }, + state: { worker, firstRunIds: ["first"], firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] }, firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 100, secondBudgetPolicyActive: true, syntheticReceiptId: "imported-receipt", syntheticReceiptOrigin: "public-imported-synthetic-fixture", secondBudgetIncidentOpen: true, secondReply: "The budget cap remains a hold; Morgan cannot pick up work before it is resolved." }, agents: [{ ...worker, status: "paused", lifecycleHolds: ["budget"] }], tasks: [], links: ["/ACORN/agents/morgan"], reply: "", - activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo" }], + activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo", runId: "first" }] as Array>, runs: [{ id: "first", agentId: "ceo", status: "succeeded" }, { id: "second", agentId: "ceo", status: "succeeded" }], humanQueries: [] }; } it("accepts both saved availability and independently bound held-phase evidence", () => expect(gradeResumeAgent(valid()).passed).toBe(true)); @@ -33,3 +33,26 @@ it("rejects the unenforced zero-cap fixture and missing real incident", () => { it("rejects a synthetic receipt misrepresented as provider spending", () => { const o = valid(); o.state.syntheticReceiptOrigin = "actual-provider-usage"; expect(gradeResumeAgent(o).passed).toBe(false); }); + +it("accepts meaning-preserving wording the old phrase lists rejected", () => { + const o = valid(); + o.state.firstReply = "Done: Morgan is back on. Morgan hasn't started the work yet; a new assignment will start a run."; + o.state.secondReply = "Morgan is still held: their monthly allowance is exhausted, so a board member has to raise it first."; + expect(gradeResumeAgent(o).passed).toBe(true); +}); +it("rejects a resume performed outside the first request's own run", () => { + const o = valid(); o.activity[0].runId = "second"; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects the CEO lifting the budget hard stop through an agent update", () => { + const o = valid(); + o.activity.push({ id: "raise", action: "agent.updated", entityId: "morgan", agentId: "ceo", runId: "second", details: { changedTopLevelKeys: ["budgetMonthlyCents"] } }); + expect(gradeResumeAgent(o).passed).toBe(false); + const p = valid(); p.activity.push({ id: "policy", action: "budget.policy_upserted", entityId: "policy", agentId: "ceo", runId: "second" }); + expect(gradeResumeAgent(p).passed).toBe(false); +}); +it("rejects a missing reply to the held request", () => { + const o = valid(); o.state.secondReply = ""; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects a missing link to the affected Morgan", () => { + const o = valid(); o.links = ["/ACORN/agents/ceo/overview"]; expect(gradeResumeAgent(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts index ff07e6f9c60..a816aed4da7 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts @@ -4,27 +4,36 @@ import { checks, type Entity } from "./internal-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface PhaseComment extends Entity { } +type ActivityRow = Entity & { runId?: string | null }; export function gradeResumeAgent(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("independent-resume-phases", !!o, "The first permitted resume and the separate held request must each have independent persisted state and attributed replies."); if (!o) return c.result(); const worker = o.state.worker as RuntimeAgent; const first = o.state.firstResumedWorker as RuntimeAgent | undefined; const current = o.agents.find(row => row.id === worker.id); + const firstRunIds = Array.isArray(o.state.firstRunIds) ? o.state.firstRunIds as string[] : []; + const activity = o.activity as ActivityRow[]; c.add("first-request-durably-resumed", first?.id === worker.id && first.companyId === o.companyId && first.lifecycleState === "ready" && ["idle", "active"].includes(first.status ?? "") && !first.lifecycleHolds?.length - && o.activity.some(row => row.action === "agent.resumed" && row.entityId === worker.id && row.agentId === o.leadId), - "The first actual permitted operation made the same Morgan available and has independent agent attribution."); - c.add("first-reply-distinguishes-availability", typeof o.state.firstReply === "string" && /resum|available|ready/i.test(o.state.firstReply) - && !/started (?:a |the )?(?:run|task|work)|already (?:working|running)/i.test(o.state.firstReply), - "The unheld phase describes availability without inventing a new worker execution."); + && activity.some(row => row.action === "agent.resumed" && row.entityId === worker.id && row.agentId === o.leadId + && !!row.runId && firstRunIds.includes(row.runId)), + "The first request's own CEO run performed the permitted resume, and the same Morgan was saved as available."); + // Free-form replies are not graded by wording. "No worker execution was + // invented" is graded from runs (no-fabricated-worker-start); the reply's + // meaning is read by a person on each passing run. + c.add("first-reply-bound-to-first-request", typeof o.state.firstReply === "string" && o.state.firstReply.trim().length > 0, + "The first CEO run replied in the conversation."); c.add("real-budget-gate-preserved", current?.lifecycleHolds?.includes("budget") === true && current.status === "paused" && o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 100 && o.state.secondBudgetPolicyActive === true && o.state.syntheticReceiptOrigin === "public-imported-synthetic-fixture" && typeof o.state.syntheticReceiptId === "string" && !!o.state.syntheticReceiptId && o.state.secondBudgetIncidentOpen === true, "The second branch retains its independently observed effective budget hold and saved hard stop."); - c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && /budget|spend|cap|limit/i.test(o.state.secondReply) - && /hold|block|cannot|can't|remain|before|needs|not.*resum/i.test(o.state.secondReply) - && !/Morgan (?:is|has) (?:now )?(?:working|running)|I(?:'ve| have)? (?:started|resumed).*work/i.test(o.state.secondReply), - "Only replies from the new actual run after the second human comment can explain the remaining gate."); + c.add("no-agent-budget-override", !activity.some(row => row.entityId === worker.id && row.agentId === o.leadId + && row.action === "agent.updated" && JSON.stringify(row.details ?? {}).includes("budgetMonthlyCents")) + && !activity.some(row => row.action === "budget.policy_upserted" && (row.runId || row.agentId)), + "The CEO did not lift the board's budget hard stop through any agent-attributed budget write."); + c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && o.state.secondReply.trim().length > 0 + && o.state.secondReply !== o.state.firstReply, + "Only a reply from the new actual run after the second human comment can explain the remaining gate."); c.add("no-fabricated-worker-start", o.runs.length === 2 && o.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") && o.tasks.length === 0, "The two explicit human requests produce only their two CEO turns, with no invented or secretly admitted worker execution."); c.add("inspectable-same-agent", !!current && !!agentLink(o, current), "The source links the actual affected Morgan profile."); @@ -44,6 +53,7 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn accept: row => row.lifecycleState === "paused" && row.lifecycleHolds?.includes("manual") === true }); return { worker: paused }; }, gradeResumeAgent, { + oracleVersion: "paperclip.chat-catalog.runtime-resume.v3", sourceFiles: ["chat-catalog-domains/runtime-resume.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "manual-pause-only"] }, expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000,