fix(evals): grade instruction edits by saved merge and link, not wording

The edit-instructions oracle matched the saved rule and the reply against
phrase regexes. It now requires the agent's own audited save to add guidance
on top of the prior and concurrent human revisions, and a link to Morgan or
the saved revision. An unsaved rule fails whatever the reply says.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -518,7 +518,7 @@
       "chat-catalog-domains/runtime-agents.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v3",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts

@@ -129,14 +129,26 @@ it("uses actual file-mutation audits rather than the current reader's virtual re
   const late = structuredClone(o); late.activity[0]!.createdAt = "2026-10-27T08:03:00Z";
   expect(gradeRuntimeAgents(late).passed).toBe(false);
 });
-it("accepts an explicit unresolved revision conflict preserving the concurrent edit", () => {
+function savedAs(o: RuntimeObservation, content: string) {
-  const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support";
+  o.instruction!.content = content;
-  o.reply = "Morgan's instruction revision changed concurrently; the testing-rule update has an unresolved conflict.";
+  o.activity[1]!.details!.contentHash = createHash("sha256").update(content).digest("hex");
+  return o;
+}
+it("grades a reworded or reordered merged rule and reply by saved structure, not wording", () => {
+  const o = savedAs(valid("agents.edit-instructions"),
+    "## Handoff\nVerify your change with the suites that cover it and share the outcome.\n\nOriginal frontend guidance\nKeep keyboard support\n");
+  o.reply = "Morgan will now verify changes with the covering suites before every handoff; it applies from the next run.";
   expect(gradeRuntimeAgents(o).passed).toBe(true);
+  o.reply = "Done."; expect(gradeRuntimeAgents(o).passed).toBe(true);
 });
-it("rejects claimed success after a conflict left the rule unsaved", () => {
+it("rejects an unsaved rule even when the reply reports a conflict or claims success", () => {
-  const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support";
+  for (const reply of ["Morgan's instructions changed concurrently; the testing rule is not saved.", "I added the testing rule for future work."]) {
-  o.reply = "There was a revision conflict, but I have added the testing rule for future work.";
+    const o = savedAs(valid("agents.edit-instructions"), "Original frontend guidance\nKeep keyboard support");
+    o.reply = reply; expect(gradeRuntimeAgents(o).passed).toBe(false);
+  }
+});
+it("rejects a merged rule that the agent did not save itself", () => {
+  const o = valid("agents.edit-instructions"); o.activity = [o.activity[0]!];
   expect(gradeRuntimeAgents(o).passed).toBe(false);
 });
 

tests/runner-e2e/chat-catalog-domains/runtime-agents.ts

@@ -41,17 +41,19 @@ export function gradeRuntimeAgents(value: unknown) {
         "The response identifies effective permission and an authorized human route rather than blaming account setup.");
       break;
     case "agents.edit-instructions": {
+      // The rule text is model-written, so grade its saved structure rather than its wording:
+      // prior and concurrent human guidance are kept verbatim and the agent's own audited save adds to them.
       const original = o.state.originalInstruction as InstructionFile | undefined;
       const concurrent = o.state.concurrentInstruction as InstructionFile | undefined;
       const current = o.instruction;
-      const ruleSaved = !!current && /(?:run|execute)[^\n.!]*?(?:relevant|appropriate|applicable)[^\n.!]*?tests?[^\n.!]*?(?:hand|return|finish|complet)|(?:before|prior to)[^\n.!]*?(?:hand|return|finish|complet)[^\n.!]*?(?:run|execut)[^\n.!]*?tests?/i.test(current.content);
       c.add("concurrent-guidance-preserved", !!original && !!concurrent && !!current && current.content.includes(original.content)
-        && current.content.includes(String(o.state.concurrentMarker)) && o.instructionSideFile?.content === o.state.sideContent,
+        && current.content.includes(String(o.state.concurrentMarker))
+        && o.instructionSideFile?.content === o.state.sideContent,
         "The saved entry retains prior and concurrently authored human guidance, and the unrelated file is unchanged.");
-      c.add("requested-rule-or-visible-conflict", ruleSaved || /conflict|changed while|newer revision|concurrent|stale/i.test(o.reply)
-        && !/(?:I(?:'ve| have)? (?:added|saved)|successfully (?:added|saved))[^.!]*?(?:rule|instruction|test)/i.test(o.reply),
-        "The rule is durably merged or the unresolved revision conflict is visibly reported without false success.");
       const write = worker && current ? instructionFileAudit(o, worker.id, current.content, "agent") : undefined;
+      c.add("requested-rule-merged-onto-newer-revision", !!original && !!concurrent && !!current && !!write
+        && current.content.replace(original.content, "").replace(String(o.state.concurrentMarker), "").trim().length > 0,
+        "The agent's own audited save adds new guidance on top of the newer concurrent revision instead of overwriting it.");
       const human = worker && concurrent ? instructionFileAudit(o, worker.id, concurrent.content, "user") : undefined;
       c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!human && o.runs.some(run => {
         const end = (run as typeof run & { finishedAt?: string }).finishedAt;
@@ -59,9 +61,8 @@ export function gradeRuntimeAgents(value: unknown) {
           && Number.isFinite(Date.parse(run.startedAt ?? "")) && Number.isFinite(Date.parse(end ?? ""))
           && Date.parse(human.createdAt!) >= Date.parse(run.startedAt!) && Date.parse(human.createdAt!) <= Date.parse(end!);
       }), "The retained human file-mutation audit occurred within the independently observed active native run.");
-      c.add("rule-attributed-if-saved", !ruleSaved || !!write, "The exact saved file bytes have a same-agent, same-native-run file mutation audit.");
+      c.add("agent-or-revision-link", !!worker && instructionResultLink(o, worker),
-      c.add("future-use-and-agent-or-revision-link", !!worker && instructionResultLink(o, worker) && /future|next|subsequent|new (?:run|work)|conflict|revision/i.test(o.reply),
+        "The reply links Morgan's profile or the exact saved revision.");
-        "The reply links Morgan or the exact saved revision and describes future instruction use or the unresolved revision.");
       break;
     }
     case "agents.restore-instructions": {
@@ -92,6 +93,7 @@ export const runtimeAgentsCases = [
     const originalInstruction = await ctx.input.api.get<InstructionFile>(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`);
     return { worker, instructionAgentId: worker.id, originalInstruction, sideContent, concurrentMarker: `Concurrent human guidance ${ctx.input.nonce}: preserve keyboard support.` };
   }, gradeRuntimeAgents, {
+    oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v3",
     async act(ctx, state) {
       const active = await pollUntil({ label: "Active instructions update before concurrent human revision", deadlineAt: Date.now() + 60_000,
         load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running",

tests/runner-e2e/chat-scenarios/agents.yaml

@@ -86,8 +86,8 @@ scenarios:
     expected_outcome: >-
       Morgan's durable instructions gain the requested rule while retaining unrelated guidance.
       The response links to the agent or revision and explains when future work will use it. A
-      concurrent edit produces a visible conflict or a fresh merge rather than silently
+      concurrent human edit made during the run is merged: the saved result keeps the newer
-      overwriting the newer instructions.
+      instructions and adds the rule rather than silently overwriting them.
   - id: agents.restore-instructions
     capabilities:
       - agents/instructions