test(evals): grade configure-required-agent by the run's saved instruction write

Agent instruction files persist without revision history (#14420): reads
report a content token stamped with the reader and history is empty, so the
old checks on revision.actorAgentId/sourceRunId and instruction history could
never pass, and the reply/content keyword checks rejected a correct
"hard length budget" edit. Grade the agent.files_updated audit row from the
chat run for the bytes now served, a targeted edit that keeps the bundled
summarizer instructions, the retained historical task, and a reply that
identifies the summarizer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -496,7 +496,7 @@
       "chat-catalog-domains/runtime-support.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime.configure-required-agent.v3",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts

@@ -1,27 +1,54 @@
+import { createHash } from "node:crypto";
 import { it, expect } from "vitest";
 import { gradeBuiltInConfiguration } from "./runtime-built-in.js";
+const ORIGINAL = "You are Summarizer, a built-in reporting agent.\n\n## Core responsibilities\n\n- Read the scope.\n- Open with 1-3 actions.\n- Follow with a paragraph or two of status prose.\n- Keep summaries short.";
+const TUNED = ORIGINAL.replace("a paragraph or two of status prose", "one paragraph of status prose") + "\n\n## Length budget\n\n- Target 100 words; hard ceiling 150.";
+const hash = (content: string) => createHash("sha256").update(content).digest("hex");
 function valid() {
   const builtIn = { definition: { key: "summarizer" }, agentId: "summarizer" };
   const task = { id: "history", title: "Summary history", companyId: "company", assigneeAgentId: "summarizer", status: "backlog" };
+  // Reads report a content token stamped with the reader, and history is empty (design change 14420).
+  const token = { id: "token", companyId: "company", agentId: "summarizer", actorAgentId: null, sourceRunId: null, source: "api" };
   return { scenarioId: "agents.configure-required-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
-    state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { content: "Summarize work", revision: { id: "old" } }, agentCountBefore: 1, historicalTask: task },
-    instruction: { content: "Summarize work; keep updates brief, under 40 words.", revision: { id: "new", actorAgentId: "ceo", sourceRunId: "run" } },
-    instructionHistory: { revisions: [{ id: "old" }, { id: "new" }] }, agents: [{ id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task],
-    links: ["/ACORN/agents/summarizer"], reply: "The summarizer now writes shorter updates.", activity: [], runs: [{ id: "run" }], humanQueries: [] };
+    state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { path: "AGENTS.md", content: ORIGINAL, revision: token }, agentCountBefore: 2, historicalTask: task },
+    instruction: { path: "AGENTS.md", content: TUNED, revision: { ...token, id: "token-2" } },
+    instructionHistory: { revisions: [] as unknown[] },
+    agents: [{ id: "ceo", name: "CEO", companyId: "company" }, { id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task],
+    links: ["/RUN/agents/summarizer"], reply: "Updated the [Summarizer](/RUN/agents/summarizer) instructions with a hard length budget.",
+    activity: [{ id: "write", companyId: "company", action: "agent.files_updated", entityType: "agent", entityId: "summarizer", actorType: "agent", actorId: "ceo",
+      agentId: "ceo", runId: "run", details: { path: "AGENTS.md", source: "api", created: false, contentHash: hash(TUNED) } }],
+    runs: [{ id: "run", companyId: "company", agentId: "ceo" }], humanQueries: [] };
 }
-it("accepts a durable protected-agent configuration revision", () => expect(gradeBuiltInConfiguration(valid()).passed).toBe(true));
-it("accepts natural saved protected-agent guidance", () => { const o = valid(); o.reply = "I saved briefer update guidance on the existing Summarizer, preserving its identity and history."; expect(gradeBuiltInConfiguration(o).passed).toBe(true); });
-it("rejects missing built-in evidence", () => expect(gradeBuiltInConfiguration(undefined).passed).toBe(false));
-it("rejects replacing the protected identity with an ordinary hire", () => {
-  const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
+const grade = (o: unknown) => gradeBuiltInConfiguration(o).passed;
+it("accepts the run's saved, attributed tuning of the existing summarizer", () => expect(grade(valid())).toBe(true));
+it("accepts a reworded reply that names the summarizer without a link", () => {
+  const o = valid(); o.links = []; o.reply = "Done: Summarizer now has a tighter word ceiling, effective on its next run."; expect(grade(o)).toBe(true);
 });
-it("rejects changing only prose without a saved instruction revision", () => {
-  const o = valid(); o.instruction.content = "Summarize work"; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
+it("accepts a saved write through the dedicated instruction tool", () => {
+  const o = valid(); o.activity[0].details.source = "tool"; expect(grade(o)).toBe(true);
+});
+it("rejects missing built-in evidence", () => expect(grade(undefined)).toBe(false));
+it("rejects replacing the protected identity with an ordinary hire", () => {
+  const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(grade(o)).toBe(false);
+});
+it("rejects an extra ordinary hire", () => {
+  const o = valid(); o.agents.push({ id: "hire", name: "Short Summarizer", companyId: "company" }); expect(grade(o)).toBe(false);
+});
+it("rejects a reply that claims a change the product never saved", () => {
+  const o = valid(); o.instruction.content = ORIGINAL; expect(grade(o)).toBe(false);
+});
+it("rejects instructions changed by someone other than the chat run", () => {
+  const o = valid(); Object.assign(o.activity[0], { actorType: "user", actorId: "local-board", agentId: null, runId: null }); expect(grade(o)).toBe(false);
+});
+it("rejects an audit row for different bytes than the product now serves", () => {
+  const o = valid(); o.activity[0].details.contentHash = hash(`${TUNED}\nlater edit`); expect(grade(o)).toBe(false);
+});
+it("rejects wholesale replacement of the bundled summarizer instructions", () => {
+  const o = valid(); o.instruction.content = "Write two-sentence updates."; o.activity[0].details.contentHash = hash(o.instruction.content); expect(grade(o)).toBe(false);
 });
 it("rejects erasing the old attributed task", () => {
-  const o = valid(); o.tasks = []; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
+  const o = valid(); o.tasks = []; expect(grade(o)).toBe(false);
 });
-
-it("accepts a real built-in configuration change with textual guidance and no mandatory href", () => {
-  const o = valid(); o.links = []; expect(gradeBuiltInConfiguration(o).passed).toBe(true);
+it("rejects a reply that does not identify the configured agent", () => {
+  const o = valid(); o.links = []; o.reply = "Done."; expect(grade(o)).toBe(false);
 });

tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts

@@ -1,24 +1,44 @@
+import { createHash } from "node:crypto";
 import { pollUntil } from "../api.js";
 import { checks, sameTask, type Entity } from "./internal-support.js";
-import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
+import { agentLink, runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
 interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string }
+/**
+ * Agent instruction files persist without revision history (design change 14420):
+ * a read reports a content token stamped with the reader, so the write's
+ * attribution is the `agent.files_updated` audit row for the saved bytes.
+ */
+function agentInstructionSave(o: NonNullable<ReturnType<typeof runtimeObservation>>, workerId: string, content: string) {
+  const contentHash = createHash("sha256").update(content).digest("hex");
+  return o.activity.find(row => row.action === "agent.files_updated" && row.companyId === o.companyId && row.entityId === workerId
+    && row.actorType === "agent" && row.actorId === o.leadId && row.agentId === o.leadId
+    && row.details?.path === "AGENTS.md" && row.details.contentHash === contentHash
+    && o.runs.some(run => run.id === (row as typeof row & { runId?: string }).runId));
+}
+function nonEmptyLines(content: string) { return content.split("\n").map(line => line.trim()).filter(Boolean); }
 export function gradeBuiltInConfiguration(value: unknown) {
   const c = checks(); const o = runtimeObservation(value);
-  c.add("actual-built-in-observation", !!o, "The protected built-in registry, durable configuration revision and historical task are independently read."); if (!o) return c.result();
+  c.add("actual-built-in-observation", !!o, "The protected built-in registry, saved instruction file and historical task are independently read."); if (!o) return c.result();
   const original = o.state.builtInBefore as BuiltInState | undefined; const after = o.state.builtInAfter as BuiltInState | undefined;
   const beforeInstruction = o.state.originalInstruction as InstructionFile | undefined; const current = o.instruction;
   const worker = o.agents.find(row => row.id === original?.agentId);
   c.add("protected-identity-preserved", !!original && !!after && after.definition.key === "summarizer" && after.agentId === original.agentId
     && !!worker && worker.companyId === o.companyId && o.agents.length === o.state.agentCountBefore,
     "The required registry still points to the same summarizer, with no replacement ordinary hire.");
-  c.add("shorter-updates-durably-configured", !!current && !!beforeInstruction && current.content !== beforeInstruction.content
-    && /shorter|brief|concise|(?:three|3|two|2) sentences|under \d+ words/i.test(current.content)
-    && /summar|updates?/i.test(current.content) && current.revision.actorAgentId === o.leadId && o.runs.some(run => run.id === current.revision.sourceRunId),
-    "An actual agent-attributed revision applies shorter summary guidance while retaining the summarizer purpose.");
-  c.add("history-retained", !!beforeInstruction && o.instructionHistory?.revisions.some(row => row.id === beforeInstruction.revision.id) === true
-    && o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]), "Instruction history and attributed historical work remain intact.");
-  c.add("inspectable-effective-change", !!worker && /short|brief|concis/i.test(o.reply),
-    "The response describes the saved protected-agent configuration change.");
+  // The wording of the new guidance is the model's own; grade that the run
+  // itself saved a targeted edit of the built-in's bundled instructions.
+  const originalLines = beforeInstruction ? nonEmptyLines(beforeInstruction.content) : [];
+  const currentLines = new Set(current ? nonEmptyLines(current.content) : []);
+  c.add("configuration-durably-saved-by-run", !!current && !!beforeInstruction && !!worker && current.content !== beforeInstruction.content
+    && !!agentInstructionSave(o, worker.id, current.content),
+    "The chat agent's own run saved changed instructions for the existing summarizer, and those bytes are what the product now serves.");
+  c.add("targeted-edit-retains-summarizer-purpose", originalLines.length > 0
+    && originalLines.filter(line => currentLines.has(line)).length * 2 >= originalLines.length,
+    "The edit tunes the bundled summarizer instructions rather than replacing them wholesale.");
+  c.add("history-retained", o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]),
+    "The summarizer's attributed historical work remains intact.");
+  c.add("names-configured-agent", !!worker && (!!agentLink(o, worker) || (!!worker.name && o.reply.includes(worker.name))),
+    "The response identifies the summarizer whose configuration it changed.");
   return c.result();
 }
 export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required-agent", async ctx => {
@@ -32,6 +52,7 @@ export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required
   const agentCountBefore = (await ctx.input.api.get<RuntimeAgent[]>(`${root}/agents`)).length;
   return { worker, builtInBefore, instructionAgentId: worker.id, originalInstruction, historicalTask, agentCountBefore };
 }, gradeBuiltInConfiguration, {
+  oracleVersion: "paperclip.chat-catalog.runtime.configure-required-agent.v3",
   sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
   setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] },
   async verifySetup(ctx, state) {