diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 70ef3ade4f5..46dd8679aaa 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -496,7 +496,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.configure-required-agent.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts index 81cccd5a5b6..f2e79c29b7b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts @@ -1,27 +1,54 @@ +import { createHash } from "node:crypto"; import { it, expect } from "vitest"; import { gradeBuiltInConfiguration } from "./runtime-built-in.js"; +const ORIGINAL = "You are Summarizer, a built-in reporting agent.\n\n## Core responsibilities\n\n- Read the scope.\n- Open with 1-3 actions.\n- Follow with a paragraph or two of status prose.\n- Keep summaries short."; +const TUNED = ORIGINAL.replace("a paragraph or two of status prose", "one paragraph of status prose") + "\n\n## Length budget\n\n- Target 100 words; hard ceiling 150."; +const hash = (content: string) => createHash("sha256").update(content).digest("hex"); function valid() { const builtIn = { definition: { key: "summarizer" }, agentId: "summarizer" }; const task = { id: "history", title: "Summary history", companyId: "company", assigneeAgentId: "summarizer", status: "backlog" }; + // Reads report a content token stamped with the reader, and history is empty (design change 14420). + const token = { id: "token", companyId: "company", agentId: "summarizer", actorAgentId: null, sourceRunId: null, source: "api" }; return { scenarioId: "agents.configure-required-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { content: "Summarize work", revision: { id: "old" } }, agentCountBefore: 1, historicalTask: task }, - instruction: { content: "Summarize work; keep updates brief, under 40 words.", revision: { id: "new", actorAgentId: "ceo", sourceRunId: "run" } }, - instructionHistory: { revisions: [{ id: "old" }, { id: "new" }] }, agents: [{ id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task], - links: ["/ACORN/agents/summarizer"], reply: "The summarizer now writes shorter updates.", activity: [], runs: [{ id: "run" }], humanQueries: [] }; + state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { path: "AGENTS.md", content: ORIGINAL, revision: token }, agentCountBefore: 2, historicalTask: task }, + instruction: { path: "AGENTS.md", content: TUNED, revision: { ...token, id: "token-2" } }, + instructionHistory: { revisions: [] as unknown[] }, + agents: [{ id: "ceo", name: "CEO", companyId: "company" }, { id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task], + links: ["/RUN/agents/summarizer"], reply: "Updated the [Summarizer](/RUN/agents/summarizer) instructions with a hard length budget.", + activity: [{ id: "write", companyId: "company", action: "agent.files_updated", entityType: "agent", entityId: "summarizer", actorType: "agent", actorId: "ceo", + agentId: "ceo", runId: "run", details: { path: "AGENTS.md", source: "api", created: false, contentHash: hash(TUNED) } }], + runs: [{ id: "run", companyId: "company", agentId: "ceo" }], humanQueries: [] }; } -it("accepts a durable protected-agent configuration revision", () => expect(gradeBuiltInConfiguration(valid()).passed).toBe(true)); -it("accepts natural saved protected-agent guidance", () => { const o = valid(); o.reply = "I saved briefer update guidance on the existing Summarizer, preserving its identity and history."; expect(gradeBuiltInConfiguration(o).passed).toBe(true); }); -it("rejects missing built-in evidence", () => expect(gradeBuiltInConfiguration(undefined).passed).toBe(false)); -it("rejects replacing the protected identity with an ordinary hire", () => { - const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(gradeBuiltInConfiguration(o).passed).toBe(false); +const grade = (o: unknown) => gradeBuiltInConfiguration(o).passed; +it("accepts the run's saved, attributed tuning of the existing summarizer", () => expect(grade(valid())).toBe(true)); +it("accepts a reworded reply that names the summarizer without a link", () => { + const o = valid(); o.links = []; o.reply = "Done: Summarizer now has a tighter word ceiling, effective on its next run."; expect(grade(o)).toBe(true); }); -it("rejects changing only prose without a saved instruction revision", () => { - const o = valid(); o.instruction.content = "Summarize work"; expect(gradeBuiltInConfiguration(o).passed).toBe(false); +it("accepts a saved write through the dedicated instruction tool", () => { + const o = valid(); o.activity[0].details.source = "tool"; expect(grade(o)).toBe(true); +}); +it("rejects missing built-in evidence", () => expect(grade(undefined)).toBe(false)); +it("rejects replacing the protected identity with an ordinary hire", () => { + const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(grade(o)).toBe(false); +}); +it("rejects an extra ordinary hire", () => { + const o = valid(); o.agents.push({ id: "hire", name: "Short Summarizer", companyId: "company" }); expect(grade(o)).toBe(false); +}); +it("rejects a reply that claims a change the product never saved", () => { + const o = valid(); o.instruction.content = ORIGINAL; expect(grade(o)).toBe(false); +}); +it("rejects instructions changed by someone other than the chat run", () => { + const o = valid(); Object.assign(o.activity[0], { actorType: "user", actorId: "local-board", agentId: null, runId: null }); expect(grade(o)).toBe(false); +}); +it("rejects an audit row for different bytes than the product now serves", () => { + const o = valid(); o.activity[0].details.contentHash = hash(`${TUNED}\nlater edit`); expect(grade(o)).toBe(false); +}); +it("rejects wholesale replacement of the bundled summarizer instructions", () => { + const o = valid(); o.instruction.content = "Write two-sentence updates."; o.activity[0].details.contentHash = hash(o.instruction.content); expect(grade(o)).toBe(false); }); it("rejects erasing the old attributed task", () => { - const o = valid(); o.tasks = []; expect(gradeBuiltInConfiguration(o).passed).toBe(false); + const o = valid(); o.tasks = []; expect(grade(o)).toBe(false); }); - -it("accepts a real built-in configuration change with textual guidance and no mandatory href", () => { - const o = valid(); o.links = []; expect(gradeBuiltInConfiguration(o).passed).toBe(true); +it("rejects a reply that does not identify the configured agent", () => { + const o = valid(); o.links = []; o.reply = "Done."; expect(grade(o)).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts index 06e23d3ff23..0374225d8cd 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts @@ -1,24 +1,44 @@ +import { createHash } from "node:crypto"; import { pollUntil } from "../api.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; -import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; +import { agentLink, runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string } +/** + * Agent instruction files persist without revision history (design change 14420): + * a read reports a content token stamped with the reader, so the write's + * attribution is the `agent.files_updated` audit row for the saved bytes. + */ +function agentInstructionSave(o: NonNullable>, workerId: string, content: string) { + const contentHash = createHash("sha256").update(content).digest("hex"); + return o.activity.find(row => row.action === "agent.files_updated" && row.companyId === o.companyId && row.entityId === workerId + && row.actorType === "agent" && row.actorId === o.leadId && row.agentId === o.leadId + && row.details?.path === "AGENTS.md" && row.details.contentHash === contentHash + && o.runs.some(run => run.id === (row as typeof row & { runId?: string }).runId)); +} +function nonEmptyLines(content: string) { return content.split("\n").map(line => line.trim()).filter(Boolean); } export function gradeBuiltInConfiguration(value: unknown) { const c = checks(); const o = runtimeObservation(value); - c.add("actual-built-in-observation", !!o, "The protected built-in registry, durable configuration revision and historical task are independently read."); if (!o) return c.result(); + c.add("actual-built-in-observation", !!o, "The protected built-in registry, saved instruction file and historical task are independently read."); if (!o) return c.result(); const original = o.state.builtInBefore as BuiltInState | undefined; const after = o.state.builtInAfter as BuiltInState | undefined; const beforeInstruction = o.state.originalInstruction as InstructionFile | undefined; const current = o.instruction; const worker = o.agents.find(row => row.id === original?.agentId); c.add("protected-identity-preserved", !!original && !!after && after.definition.key === "summarizer" && after.agentId === original.agentId && !!worker && worker.companyId === o.companyId && o.agents.length === o.state.agentCountBefore, "The required registry still points to the same summarizer, with no replacement ordinary hire."); - c.add("shorter-updates-durably-configured", !!current && !!beforeInstruction && current.content !== beforeInstruction.content - && /shorter|brief|concise|(?:three|3|two|2) sentences|under \d+ words/i.test(current.content) - && /summar|updates?/i.test(current.content) && current.revision.actorAgentId === o.leadId && o.runs.some(run => run.id === current.revision.sourceRunId), - "An actual agent-attributed revision applies shorter summary guidance while retaining the summarizer purpose."); - c.add("history-retained", !!beforeInstruction && o.instructionHistory?.revisions.some(row => row.id === beforeInstruction.revision.id) === true - && o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]), "Instruction history and attributed historical work remain intact."); - c.add("inspectable-effective-change", !!worker && /short|brief|concis/i.test(o.reply), - "The response describes the saved protected-agent configuration change."); + // The wording of the new guidance is the model's own; grade that the run + // itself saved a targeted edit of the built-in's bundled instructions. + const originalLines = beforeInstruction ? nonEmptyLines(beforeInstruction.content) : []; + const currentLines = new Set(current ? nonEmptyLines(current.content) : []); + c.add("configuration-durably-saved-by-run", !!current && !!beforeInstruction && !!worker && current.content !== beforeInstruction.content + && !!agentInstructionSave(o, worker.id, current.content), + "The chat agent's own run saved changed instructions for the existing summarizer, and those bytes are what the product now serves."); + c.add("targeted-edit-retains-summarizer-purpose", originalLines.length > 0 + && originalLines.filter(line => currentLines.has(line)).length * 2 >= originalLines.length, + "The edit tunes the bundled summarizer instructions rather than replacing them wholesale."); + c.add("history-retained", o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]), + "The summarizer's attributed historical work remains intact."); + c.add("names-configured-agent", !!worker && (!!agentLink(o, worker) || (!!worker.name && o.reply.includes(worker.name))), + "The response identifies the summarizer whose configuration it changed."); return c.result(); } export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required-agent", async ctx => { @@ -32,6 +52,7 @@ export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required const agentCountBefore = (await ctx.input.api.get(`${root}/agents`)).length; return { worker, builtInBefore, instructionAgentId: worker.id, originalInstruction, historicalTask, agentCountBefore }; }, gradeBuiltInConfiguration, { + oracleVersion: "paperclip.chat-catalog.runtime.configure-required-agent.v3", sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] }, async verifySetup(ctx, state) {