fix(evals): bind instruction edits to supported file mutation audit

tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from "vitest";
 import { gradeRuntimeAgents, hiringDeniedPermissions, runtimeAgentsCases } from "./runtime-agents.js";
 import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js";
+import { createHash } from "node:crypto";
 import type { RuntimeObservation } from "./runtime-support.js";
 
 function valid(id: string): RuntimeObservation {
@@ -28,6 +29,14 @@ function valid(id: string): RuntimeObservation {
     o.instruction = { path: "AGENTS.md", content: "Original frontend guidance\nKeep keyboard support\nRun the relevant tests before handing work back.",
       revision: { ...revision, id: "merged", source: "api", actorAgentId: "ceo", sourceRunId: "run" } };
     o.instructionHistory = { revisions: [revision, { ...revision, id: "concurrent" }, o.instruction.revision] };
+    Object.assign(o.runs[0]!, { startedAt: "2026-10-27T08:00:00Z", finishedAt: "2026-10-27T08:02:00Z" });
+    o.state.concurrentRunId = "run";
+    o.activity = [
+      { id: "human-write", companyId: "company", action: "agent.files_updated", entityId: "morgan", actorType: "user", actorId: "local-board", createdAt: "2026-10-27T08:00:30Z",
+        details: { path: "AGENTS.md", source: "board", contentHash: createHash("sha256").update("Original frontend guidance\nKeep keyboard support").digest("hex") } },
+      Object.assign({ id: "agent-write", companyId: "company", action: "agent.files_updated", entityId: "morgan", actorType: "agent", actorId: "ceo", agentId: "ceo",
+        details: { path: "AGENTS.md", source: "tool", contentHash: createHash("sha256").update(o.instruction.content).digest("hex") } }, { runId: "run" }),
+    ];
     o.reply = "Added the rule to Morgan's instructions for future work.";
   } else {
     const revision = { id: "previous", agentId: "morgan", companyId: "company", source: "seed" };
@@ -99,6 +108,22 @@ it("rejects an instruction edit that overwrote concurrent human guidance", () =>
   const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nRun the relevant tests before handing work back.";
   expect(gradeRuntimeAgents(o).passed).toBe(false);
 });
+it("uses actual file-mutation audits rather than the current reader's virtual revision actor/history", () => {
+  const o = valid("agents.edit-instructions");
+  o.instructionHistory = { revisions: [] };
+  Object.assign(o.instruction!.revision, { actorAgentId: null, sourceRunId: null, actorUserId: "local-board" });
+  expect(gradeRuntimeAgents(o).passed).toBe(true);
+  for (const change of [
+    { companyId: "foreign" }, { entityId: "other-agent" }, { actorId: "other-agent" }, { runId: "other-run" },
+    { details: { path: "other.md", source: "tool", contentHash: o.activity[1]!.details!.contentHash } },
+    { details: { path: "AGENTS.md", source: "tool", contentHash: "wrong-hash" } },
+  ]) {
+    const wrong = structuredClone(o); Object.assign(wrong.activity[1]!, change);
+    expect(gradeRuntimeAgents(wrong).passed).toBe(false);
+  }
+  const late = structuredClone(o); late.activity[0]!.createdAt = "2026-10-27T08:03:00Z";
+  expect(gradeRuntimeAgents(late).passed).toBe(false);
+});
 it("accepts an explicit unresolved revision conflict preserving the concurrent edit", () => {
   const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support";
   o.reply = "Morgan's instruction revision changed concurrently; the testing-rule update has an unresolved conflict.";

tests/runner-e2e/chat-catalog-domains/runtime-agents.ts

@@ -2,6 +2,7 @@ import { checks, sameTask } from "./internal-support.js";
 import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, agentLink, verifyRuntimeResult, type RuntimeObservation, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
 import { pollUntil } from "../api.js";
 import { expect } from "@playwright/test";
+import { createHash } from "node:crypto";
 import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js";
 
 export function hiringDeniedPermissions(permissions: RuntimeAgent["permissions"]) {
@@ -48,11 +49,15 @@ export function gradeRuntimeAgents(value: unknown) {
       c.add("requested-rule-or-visible-conflict", ruleSaved || /conflict|changed while|newer revision|concurrent|stale/i.test(o.reply)
         && !/(?:I(?:'ve| have)? (?:added|saved)|successfully (?:added|saved))[^.!]*?(?:rule|instruction|test)/i.test(o.reply),
         "The rule is durably merged or the unresolved revision conflict is visibly reported without false success.");
-      c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!concurrent
-        && o.instructionHistory?.revisions.some(row => row.id === concurrent.revision.id) === true,
-        "An actual human revision occurred during the canonical query's active native run and remains in history.");
-      c.add("rule-attributed-if-saved", !ruleSaved || !!current && current.revision.actorAgentId === o.leadId
-        && o.runs.some(run => run.id === current.revision.sourceRunId), "A saved testing rule belongs to this actual agent run.");
+      const write = worker && current ? instructionFileAudit(o, worker.id, current.content, "agent") : undefined;
+      const human = worker && concurrent ? instructionFileAudit(o, worker.id, concurrent.content, "user") : undefined;
+      c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!human && o.runs.some(run => {
+        const end = (run as typeof run & { finishedAt?: string }).finishedAt;
+        return run.agentId === o.leadId && run.companyId === o.companyId && (!o.state.concurrentRunId || run.id === o.state.concurrentRunId)
+          && Number.isFinite(Date.parse(run.startedAt ?? "")) && Number.isFinite(Date.parse(end ?? ""))
+          && Date.parse(human.createdAt!) >= Date.parse(run.startedAt!) && Date.parse(human.createdAt!) <= Date.parse(end!);
+      }), "The retained human file-mutation audit occurred within the independently observed active native run.");
+      c.add("rule-attributed-if-saved", !ruleSaved || !!write, "The exact saved file bytes have a same-agent, same-native-run file mutation audit.");
       c.add("future-use-and-agent-or-revision-link", !!worker && instructionResultLink(o, worker) && /future|next|subsequent|new (?:run|work)|conflict|revision/i.test(o.reply),
         "The reply links Morgan or the exact saved revision and describes future instruction use or the unresolved revision.");
       break;
@@ -90,6 +95,7 @@ export const runtimeAgentsCases = [
         load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running",
         reject: runs => runs.some(run => ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status)) ? "Instruction run settled before concurrent fixture boundary" : undefined });
       state.concurrentRunStatus = active[0]!.status;
+      state.concurrentRunId = active[0]!.id;
       const root = `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle/file`;
       const current = await ctx.input.api.get<InstructionFile>(`${root}?path=AGENTS.md`);
       if (current.revision.id !== (state.originalInstruction as InstructionFile).revision.id)
@@ -149,8 +155,32 @@ export const runtimeAgentsCases = [
     const unwantedInstruction = await ctx.input.api.putSensitive<InstructionFile>(root, { path: "AGENTS.md",
       content: `${previousInstruction.content}\nUnwanted rule: never run frontend tests.`, baseRevisionId: previousInstruction.revision.id });
     return { worker, instructionAgentId: worker.id, previousInstruction, unwantedInstruction, sideContent };
-  }, gradeRuntimeAgents),
+  }, gradeRuntimeAgents, {
+    coverageGaps: ["supported-two-visible-instruction-history-versions"],
+    async verifySetup(ctx, state) {
+      const { readAgentResource } = await import("./identity-support.js");
+      const history = await readAgentResource(ctx, `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle/history?path=AGENTS.md`);
+      const rows = (history.data as { revisions?: InstructionFile["revision"][] } | undefined)?.revisions;
+      const previous = state.previousInstruction as InstructionFile; const unwanted = state.unwantedInstruction as InstructionFile;
+      const c = checks();
+      c.add("two-actual-visible-prior-versions", history.status === 200 && Array.isArray(rows) && previous.revision.id !== unwanted.revision.id
+        && [previous.revision.id, unwanted.revision.id].every(id => rows.some(row => row.id === id)),
+        "The actual speaking agent must see both genuine historical versions before the unchanged restore request; content ETags and newly written files do not manufacture history.");
+      await ctx.input.evidence("runtime-instruction-history-precondition.json", { agentId: (state.worker as RuntimeAgent).id, status: history.status,
+        visibleRevisionIds: rows?.map(row => row.id) ?? [], expectedRevisionIds: [previous.revision.id, unwanted.revision.id],
+        productContract: "Current file writes use content tokens; legacy history is read-only (design change14420).", prerequisiteSatisfied: c.result().passed });
+      return c.result();
+    },
+  }),
 ];
+function instructionFileAudit(o: RuntimeObservation, workerId: string, content: string, actor: "agent" | "user") {
+  const hash = createHash("sha256").update(content).digest("hex");
+  return o.activity.find(row => row.action === "agent.files_updated" && row.companyId === o.companyId && row.entityId === workerId
+    && row.actorType === actor && row.details?.path === "AGENTS.md" && row.details.contentHash === hash
+    && (actor === "user" ? !row.agentId && row.details.source === "board" : row.agentId === o.leadId && row.actorId === o.leadId
+      && row.details.source === "tool" && o.runs.some(run => run.id === (row as typeof row & { runId?: string }).runId
+        && run.companyId === o.companyId && run.agentId === o.leadId)));
+}
 function instructionResultLink(o: RuntimeObservation, worker: RuntimeAgent): boolean {
   return !!agentLink(o, worker) || !!o.instruction && o.links.some(href => {
     try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin