diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts index bf352cee6b0..97123bd607f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; import { gradeRuntimeAgents, hiringDeniedPermissions, runtimeAgentsCases } from "./runtime-agents.js"; import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js"; +import { createHash } from "node:crypto"; import type { RuntimeObservation } from "./runtime-support.js"; function valid(id: string): RuntimeObservation { @@ -28,6 +29,14 @@ function valid(id: string): RuntimeObservation { o.instruction = { path: "AGENTS.md", content: "Original frontend guidance\nKeep keyboard support\nRun the relevant tests before handing work back.", revision: { ...revision, id: "merged", source: "api", actorAgentId: "ceo", sourceRunId: "run" } }; o.instructionHistory = { revisions: [revision, { ...revision, id: "concurrent" }, o.instruction.revision] }; + Object.assign(o.runs[0]!, { startedAt: "2026-10-27T08:00:00Z", finishedAt: "2026-10-27T08:02:00Z" }); + o.state.concurrentRunId = "run"; + o.activity = [ + { id: "human-write", companyId: "company", action: "agent.files_updated", entityId: "morgan", actorType: "user", actorId: "local-board", createdAt: "2026-10-27T08:00:30Z", + details: { path: "AGENTS.md", source: "board", contentHash: createHash("sha256").update("Original frontend guidance\nKeep keyboard support").digest("hex") } }, + Object.assign({ id: "agent-write", companyId: "company", action: "agent.files_updated", entityId: "morgan", actorType: "agent", actorId: "ceo", agentId: "ceo", + details: { path: "AGENTS.md", source: "tool", contentHash: createHash("sha256").update(o.instruction.content).digest("hex") } }, { runId: "run" }), + ]; o.reply = "Added the rule to Morgan's instructions for future work."; } else { const revision = { id: "previous", agentId: "morgan", companyId: "company", source: "seed" }; @@ -99,6 +108,22 @@ it("rejects an instruction edit that overwrote concurrent human guidance", () => const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nRun the relevant tests before handing work back."; expect(gradeRuntimeAgents(o).passed).toBe(false); }); +it("uses actual file-mutation audits rather than the current reader's virtual revision actor/history", () => { + const o = valid("agents.edit-instructions"); + o.instructionHistory = { revisions: [] }; + Object.assign(o.instruction!.revision, { actorAgentId: null, sourceRunId: null, actorUserId: "local-board" }); + expect(gradeRuntimeAgents(o).passed).toBe(true); + for (const change of [ + { companyId: "foreign" }, { entityId: "other-agent" }, { actorId: "other-agent" }, { runId: "other-run" }, + { details: { path: "other.md", source: "tool", contentHash: o.activity[1]!.details!.contentHash } }, + { details: { path: "AGENTS.md", source: "tool", contentHash: "wrong-hash" } }, + ]) { + const wrong = structuredClone(o); Object.assign(wrong.activity[1]!, change); + expect(gradeRuntimeAgents(wrong).passed).toBe(false); + } + const late = structuredClone(o); late.activity[0]!.createdAt = "2026-10-27T08:03:00Z"; + expect(gradeRuntimeAgents(late).passed).toBe(false); +}); it("accepts an explicit unresolved revision conflict preserving the concurrent edit", () => { const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support"; o.reply = "Morgan's instruction revision changed concurrently; the testing-rule update has an unresolved conflict."; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts index 1c4f50f5a02..1b3f35d6937 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts @@ -2,6 +2,7 @@ import { checks, sameTask } from "./internal-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, agentLink, verifyRuntimeResult, type RuntimeObservation, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; import { pollUntil } from "../api.js"; import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; import { updateAgentPermissionsSchema } from "../../../packages/shared/src/validators/agent.js"; export function hiringDeniedPermissions(permissions: RuntimeAgent["permissions"]) { @@ -48,11 +49,15 @@ export function gradeRuntimeAgents(value: unknown) { c.add("requested-rule-or-visible-conflict", ruleSaved || /conflict|changed while|newer revision|concurrent|stale/i.test(o.reply) && !/(?:I(?:'ve| have)? (?:added|saved)|successfully (?:added|saved))[^.!]*?(?:rule|instruction|test)/i.test(o.reply), "The rule is durably merged or the unresolved revision conflict is visibly reported without false success."); - c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!concurrent - && o.instructionHistory?.revisions.some(row => row.id === concurrent.revision.id) === true, - "An actual human revision occurred during the canonical query's active native run and remains in history."); - c.add("rule-attributed-if-saved", !ruleSaved || !!current && current.revision.actorAgentId === o.leadId - && o.runs.some(run => run.id === current.revision.sourceRunId), "A saved testing rule belongs to this actual agent run."); + const write = worker && current ? instructionFileAudit(o, worker.id, current.content, "agent") : undefined; + const human = worker && concurrent ? instructionFileAudit(o, worker.id, concurrent.content, "user") : undefined; + c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!human && o.runs.some(run => { + const end = (run as typeof run & { finishedAt?: string }).finishedAt; + return run.agentId === o.leadId && run.companyId === o.companyId && (!o.state.concurrentRunId || run.id === o.state.concurrentRunId) + && Number.isFinite(Date.parse(run.startedAt ?? "")) && Number.isFinite(Date.parse(end ?? "")) + && Date.parse(human.createdAt!) >= Date.parse(run.startedAt!) && Date.parse(human.createdAt!) <= Date.parse(end!); + }), "The retained human file-mutation audit occurred within the independently observed active native run."); + c.add("rule-attributed-if-saved", !ruleSaved || !!write, "The exact saved file bytes have a same-agent, same-native-run file mutation audit."); c.add("future-use-and-agent-or-revision-link", !!worker && instructionResultLink(o, worker) && /future|next|subsequent|new (?:run|work)|conflict|revision/i.test(o.reply), "The reply links Morgan or the exact saved revision and describes future instruction use or the unresolved revision."); break; @@ -90,6 +95,7 @@ export const runtimeAgentsCases = [ load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running", reject: runs => runs.some(run => ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status)) ? "Instruction run settled before concurrent fixture boundary" : undefined }); state.concurrentRunStatus = active[0]!.status; + state.concurrentRunId = active[0]!.id; const root = `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle/file`; const current = await ctx.input.api.get(`${root}?path=AGENTS.md`); if (current.revision.id !== (state.originalInstruction as InstructionFile).revision.id) @@ -149,8 +155,32 @@ export const runtimeAgentsCases = [ const unwantedInstruction = await ctx.input.api.putSensitive(root, { path: "AGENTS.md", content: `${previousInstruction.content}\nUnwanted rule: never run frontend tests.`, baseRevisionId: previousInstruction.revision.id }); return { worker, instructionAgentId: worker.id, previousInstruction, unwantedInstruction, sideContent }; - }, gradeRuntimeAgents), + }, gradeRuntimeAgents, { + coverageGaps: ["supported-two-visible-instruction-history-versions"], + async verifySetup(ctx, state) { + const { readAgentResource } = await import("./identity-support.js"); + const history = await readAgentResource(ctx, `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle/history?path=AGENTS.md`); + const rows = (history.data as { revisions?: InstructionFile["revision"][] } | undefined)?.revisions; + const previous = state.previousInstruction as InstructionFile; const unwanted = state.unwantedInstruction as InstructionFile; + const c = checks(); + c.add("two-actual-visible-prior-versions", history.status === 200 && Array.isArray(rows) && previous.revision.id !== unwanted.revision.id + && [previous.revision.id, unwanted.revision.id].every(id => rows.some(row => row.id === id)), + "The actual speaking agent must see both genuine historical versions before the unchanged restore request; content ETags and newly written files do not manufacture history."); + await ctx.input.evidence("runtime-instruction-history-precondition.json", { agentId: (state.worker as RuntimeAgent).id, status: history.status, + visibleRevisionIds: rows?.map(row => row.id) ?? [], expectedRevisionIds: [previous.revision.id, unwanted.revision.id], + productContract: "Current file writes use content tokens; legacy history is read-only (design change14420).", prerequisiteSatisfied: c.result().passed }); + return c.result(); + }, + }), ]; +function instructionFileAudit(o: RuntimeObservation, workerId: string, content: string, actor: "agent" | "user") { + const hash = createHash("sha256").update(content).digest("hex"); + return o.activity.find(row => row.action === "agent.files_updated" && row.companyId === o.companyId && row.entityId === workerId + && row.actorType === actor && row.details?.path === "AGENTS.md" && row.details.contentHash === hash + && (actor === "user" ? !row.agentId && row.details.source === "board" : row.agentId === o.leadId && row.actorId === o.leadId + && row.details.source === "tool" && o.runs.some(run => run.id === (row as typeof row & { runId?: string }).runId + && run.companyId === o.companyId && run.agentId === o.leadId))); +} function instructionResultLink(o: RuntimeObservation, worker: RuntimeAgent): boolean { return !!agentLink(o, worker) || !!o.instruction && o.links.some(href => { try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin