diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 5b9741fc38c..e395ae84384 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -518,7 +518,7 @@ "chat-catalog-domains/runtime-agents.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts index 11e2466bd10..bdeff85272a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts @@ -129,14 +129,26 @@ it("uses actual file-mutation audits rather than the current reader's virtual re const late = structuredClone(o); late.activity[0]!.createdAt = "2026-10-27T08:03:00Z"; expect(gradeRuntimeAgents(late).passed).toBe(false); }); -it("accepts an explicit unresolved revision conflict preserving the concurrent edit", () => { - const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support"; - o.reply = "Morgan's instruction revision changed concurrently; the testing-rule update has an unresolved conflict."; +function savedAs(o: RuntimeObservation, content: string) { + o.instruction!.content = content; + o.activity[1]!.details!.contentHash = createHash("sha256").update(content).digest("hex"); + return o; +} +it("grades a reworded or reordered merged rule and reply by saved structure, not wording", () => { + const o = savedAs(valid("agents.edit-instructions"), + "## Handoff\nVerify your change with the suites that cover it and share the outcome.\n\nOriginal frontend guidance\nKeep keyboard support\n"); + o.reply = "Morgan will now verify changes with the covering suites before every handoff; it applies from the next run."; expect(gradeRuntimeAgents(o).passed).toBe(true); + o.reply = "Done."; expect(gradeRuntimeAgents(o).passed).toBe(true); }); -it("rejects claimed success after a conflict left the rule unsaved", () => { - const o = valid("agents.edit-instructions"); o.instruction!.content = "Original frontend guidance\nKeep keyboard support"; - o.reply = "There was a revision conflict, but I have added the testing rule for future work."; +it("rejects an unsaved rule even when the reply reports a conflict or claims success", () => { + for (const reply of ["Morgan's instructions changed concurrently; the testing rule is not saved.", "I added the testing rule for future work."]) { + const o = savedAs(valid("agents.edit-instructions"), "Original frontend guidance\nKeep keyboard support"); + o.reply = reply; expect(gradeRuntimeAgents(o).passed).toBe(false); + } +}); +it("rejects a merged rule that the agent did not save itself", () => { + const o = valid("agents.edit-instructions"); o.activity = [o.activity[0]!]; expect(gradeRuntimeAgents(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts index b79f9a37fe9..dcc24703fa7 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts @@ -41,17 +41,19 @@ export function gradeRuntimeAgents(value: unknown) { "The response identifies effective permission and an authorized human route rather than blaming account setup."); break; case "agents.edit-instructions": { + // The rule text is model-written, so grade its saved structure rather than its wording: + // prior and concurrent human guidance are kept verbatim and the agent's own audited save adds to them. const original = o.state.originalInstruction as InstructionFile | undefined; const concurrent = o.state.concurrentInstruction as InstructionFile | undefined; const current = o.instruction; - const ruleSaved = !!current && /(?:run|execute)[^\n.!]*?(?:relevant|appropriate|applicable)[^\n.!]*?tests?[^\n.!]*?(?:hand|return|finish|complet)|(?:before|prior to)[^\n.!]*?(?:hand|return|finish|complet)[^\n.!]*?(?:run|execut)[^\n.!]*?tests?/i.test(current.content); c.add("concurrent-guidance-preserved", !!original && !!concurrent && !!current && current.content.includes(original.content) - && current.content.includes(String(o.state.concurrentMarker)) && o.instructionSideFile?.content === o.state.sideContent, + && current.content.includes(String(o.state.concurrentMarker)) + && o.instructionSideFile?.content === o.state.sideContent, "The saved entry retains prior and concurrently authored human guidance, and the unrelated file is unchanged."); - c.add("requested-rule-or-visible-conflict", ruleSaved || /conflict|changed while|newer revision|concurrent|stale/i.test(o.reply) - && !/(?:I(?:'ve| have)? (?:added|saved)|successfully (?:added|saved))[^.!]*?(?:rule|instruction|test)/i.test(o.reply), - "The rule is durably merged or the unresolved revision conflict is visibly reported without false success."); const write = worker && current ? instructionFileAudit(o, worker.id, current.content, "agent") : undefined; + c.add("requested-rule-merged-onto-newer-revision", !!original && !!concurrent && !!current && !!write + && current.content.replace(original.content, "").replace(String(o.state.concurrentMarker), "").trim().length > 0, + "The agent's own audited save adds new guidance on top of the newer concurrent revision instead of overwriting it."); const human = worker && concurrent ? instructionFileAudit(o, worker.id, concurrent.content, "user") : undefined; c.add("actual-concurrent-boundary", o.state.concurrentRunStatus === "running" && !!human && o.runs.some(run => { const end = (run as typeof run & { finishedAt?: string }).finishedAt; @@ -59,9 +61,8 @@ export function gradeRuntimeAgents(value: unknown) { && Number.isFinite(Date.parse(run.startedAt ?? "")) && Number.isFinite(Date.parse(end ?? "")) && Date.parse(human.createdAt!) >= Date.parse(run.startedAt!) && Date.parse(human.createdAt!) <= Date.parse(end!); }), "The retained human file-mutation audit occurred within the independently observed active native run."); - c.add("rule-attributed-if-saved", !ruleSaved || !!write, "The exact saved file bytes have a same-agent, same-native-run file mutation audit."); - c.add("future-use-and-agent-or-revision-link", !!worker && instructionResultLink(o, worker) && /future|next|subsequent|new (?:run|work)|conflict|revision/i.test(o.reply), - "The reply links Morgan or the exact saved revision and describes future instruction use or the unresolved revision."); + c.add("agent-or-revision-link", !!worker && instructionResultLink(o, worker), + "The reply links Morgan's profile or the exact saved revision."); break; } case "agents.restore-instructions": { @@ -92,6 +93,7 @@ export const runtimeAgentsCases = [ const originalInstruction = await ctx.input.api.get(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`); return { worker, instructionAgentId: worker.id, originalInstruction, sideContent, concurrentMarker: `Concurrent human guidance ${ctx.input.nonce}: preserve keyboard support.` }; }, gradeRuntimeAgents, { + oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v3", async act(ctx, state) { const active = await pollUntil({ label: "Active instructions update before concurrent human revision", deadlineAt: Date.now() + 60_000, load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running", diff --git a/tests/runner-e2e/chat-scenarios/agents.yaml b/tests/runner-e2e/chat-scenarios/agents.yaml index 821bb3a4561..5be26ac3c5e 100644 --- a/tests/runner-e2e/chat-scenarios/agents.yaml +++ b/tests/runner-e2e/chat-scenarios/agents.yaml @@ -86,8 +86,8 @@ scenarios: expected_outcome: >- Morgan's durable instructions gain the requested rule while retaining unrelated guidance. The response links to the agent or revision and explains when future work will use it. A - concurrent edit produces a visible conflict or a fresh merge rather than silently - overwriting the newer instructions. + concurrent human edit made during the run is merged: the saved result keeps the newer + instructions and adds the rule rather than silently overwriting them. - id: agents.restore-instructions capabilities: - agents/instructions