From 657c629679c8a95e32f0198d800a25fc7eb53521 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:26:34 -0600 Subject: [PATCH] test(evals): grade find-runs on its run-history read and exact run links, not reply wording The interrupted-and-continuation-states-explained check matched state words on reply lines that contained a raw run UUID plus the literal "Morgan". Correct replies that link runs on a separate line, or use other state words, failed; it also pushed the agent toward pasting raw IDs. Replace it with a check that the chat run read the website task's run history (GET /api/issues/{id}/runs or the exact run detail) through a product tool. Exact native run links per agent, and opening them to the real cancelled/succeeded status, remain required. Bump the oracle to paperclip.chat-catalog.run-history.v2. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 5 ++- .../internal-run-history.test.ts | 18 +++++++-- .../internal-run-history.ts | 39 +++++++++++++++---- .../chat-catalog-domains/internal.test.ts | 2 +- 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b4a90ddc409..9ba9f57077a 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -203,9 +203,10 @@ "chat-scenarios/activity.yaml", "chat-catalog-domains/internal-run-history.ts", "chat-catalog-domains/internal-run-brief.ts", - "../e2e/fetch-allowed-port.ts" + "../e2e/fetch-allowed-port.ts", + "run-observations.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.run-history.v2", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-run-history.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-run-history.test.ts index 46d38b37208..cbe9f6641c2 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-run-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-run-history.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradeRunHistory } from "./internal-run-history.js"; +import { gradeRunHistory, readTaskRunHistory } from "./internal-run-history.js"; function valid() { const task = { id: "task", title: "Website task", status: "done", assigneeAgentId: "morgan" }; const interruptedRun = { id: "interrupted", companyId: "company", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "task" } }; @@ -7,14 +7,26 @@ function valid() { return { scenarioId: "activity.find-runs", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task, worker: { id: "morgan" }, interruptedRun, continuationRun, interruptedTask: { ...task, status: "in_progress", executionRunId: "interrupted" }, gateCalls: [{ authorized: true, operation: "read_website_brief" }] }, tasks: [task], projects: [], goals: [], comments: [], activity: [] as Array<{ entityId: string; agentId: string; action: string }>, documents: [], revisions: [], humanQueries: [], reply: "Morgan's interrupted attempt [cancelled](/RUN/agents/morgan/runs/interrupted).\nMorgan's continuation [succeeded](/RUN/agents/morgan/runs/continuation).", links: ["/RUN/issues/task", "/RUN/agents/morgan/runs/interrupted", "/RUN/agents/morgan/runs/continuation"], - runs: [interruptedRun, continuationRun], agentRunReadStatuses: [200, 200], currentTask: task }; + runs: [interruptedRun, continuationRun], agentRunReadStatuses: [200, 200], runHistoryRead: true, currentTask: task }; } describe("actual interrupted and continued execution calibration", () => { it("accepts independently retained cancellation and successful continuation", () => expect(gradeRunHistory(valid()).passed).toBe(true)); it("rejects exposing a run whose acting-agent detail read is denied", () => { const o = valid(); o.agentRunReadStatuses[0] = 403; expect(gradeRunHistory(o).passed).toBe(false); }); it("rejects hiding the interrupted attempt", () => { const o = valid(); o.runs = o.runs.slice(1); expect(gradeRunHistory(o).passed).toBe(false); }); it("rejects the wrong execution owner or task binding", () => { const o = valid(); o.runs[0]!.agentId = "riley"; expect(gradeRunHistory(o).passed).toBe(false); const wrong = valid(); wrong.runs[1]!.contextSnapshot.issueId = "another"; expect(gradeRunHistory(wrong).passed).toBe(false); }); - it("rejects swapped reply states despite both words elsewhere", () => { const o = valid(); o.reply = "Morgan interrupted succeeded.\nMorgan continuation cancelled."; expect(gradeRunHistory(o).passed).toBe(false); }); + it("does not match reply wording", () => { const o = valid(); o.reply = `Attempt one [open](${o.links[1]}) stopped early.\nAttempt two [open](${o.links[2]}) finished the website task.`; expect(gradeRunHistory(o).passed).toBe(true); }); + it("rejects a reply that never read the task's run history", () => { const o = valid(); o.runHistoryRead = false; expect(gradeRunHistory(o).passed).toBe(false); }); + it("recognizes only a completed runner read of this task's runs or an exact run detail", () => { + const event = (output: string, status = "completed") => ({ eventType: "tool.execution.completed", payload: { prpEvent: { sourceKind: "runner", payload: { status, output } } } }); + const issueRuns = JSON.stringify({ text: JSON.stringify({ result: { apiOperationId: "GET /api/issues/{id}/runs", data: [{ contextIssueId: "task", runId: "interrupted" }] } }) }); + const detail = JSON.stringify({ text: JSON.stringify({ result: { apiOperationId: "GET /api/heartbeat-runs/{runId}", data: { id: "continuation" } } }) }); + expect(readTaskRunHistory([event(issueRuns)], "task", ["interrupted", "continuation"])).toBe(true); + expect(readTaskRunHistory([event(detail)], "task", ["interrupted", "continuation"])).toBe(true); + expect(readTaskRunHistory([event(issueRuns)], "another", ["other"])).toBe(false); + expect(readTaskRunHistory([event(issueRuns, "failed")], "task", ["interrupted"])).toBe(false); + expect(readTaskRunHistory([event(JSON.stringify({ text: "GET /api/issues/{id}/runs task" }))], "task", ["interrupted"])).toBe(false); + expect(readTaskRunHistory([event(detail.replace("{runId}", "{runId}/events"))], "task", ["continuation"])).toBe(false); + }); it("rejects foreign or missing execution links", () => { const o = valid(); o.links[1] = "https://foreign.test/RUN/agents/morgan/runs/interrupted"; expect(gradeRunHistory(o).passed).toBe(false); }); it("requires actual held checkout and authorized app call", () => { const o = valid(); o.state.interruptedTask.executionRunId = "other"; expect(gradeRunHistory(o).passed).toBe(false); const call = valid(); call.state.gateCalls = []; expect(gradeRunHistory(call).passed).toBe(false); }); it("rejects changing task history or ownership while listing", () => { const o = valid(); o.activity.push({ entityId: "task", agentId: "ceo", action: "issue.updated" }); expect(gradeRunHistory(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-run-history.ts b/tests/runner-e2e/chat-catalog-domains/internal-run-history.ts index ac358d1f43f..46bb852d06a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-run-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-run-history.ts @@ -2,11 +2,27 @@ import { randomUUID } from "node:crypto"; import { expect } from "@playwright/test"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; +import { collectRunEvents } from "../run-observations.js"; import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { startRunBriefGate } from "./internal-run-brief.js"; import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js"; interface OwnedTask extends Entity { checkoutRunId?: string | null; executionRunId?: string | null } -interface RunHistory extends InternalObservation { currentTask: OwnedTask; agentRunReadStatuses: number[] } +interface RunHistory extends InternalObservation { currentTask: OwnedTask; agentRunReadStatuses: number[]; runHistoryRead?: boolean } +const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +/** + * The chat run read the task's execution history through a product tool before + * answering. Tool outputs are product data, so they are matched exactly; the + * agent's own wording is never matched. + */ +export function readTaskRunHistory(events: readonly Record[], taskId: string, runIds: readonly string[]) { + return !!taskId && events.some(row => { + const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload); + const output = typeof payload.output === "string" ? payload.output : ""; + return row.eventType === "tool.execution.completed" && envelope.sourceKind === "runner" && payload.status === "completed" + && (/apiOperationId\\*"\s*:\s*\\*"GET \/api\/issues\/\{id\}\/runs(?![\w/{])/.test(output) && output.includes(taskId) + || /apiOperationId\\*"\s*:\s*\\*"GET \/api\/heartbeat-runs\/\{runId\}(?![\w/{])/.test(output) && runIds.some(id => !!id && output.includes(id))); + }); +} function runHref(o: InternalObservation, agentId: string, runId: string) { return o.links.find(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.endsWith(`/agents/${agentId}/runs/${runId}`); } catch { return false; } }); } @@ -24,10 +40,11 @@ export function gradeRunHistory(value: unknown) { c.add("independent-agent-execution-read-authority", Array.isArray(o.agentRunReadStatuses) && o.agentRunReadStatuses.length === 2 && o.agentRunReadStatuses.every(status => status === 200), "The actual acting agent can read both exact run details under its own bearer; task access alone is not assumed to grant run access."); c.add("exact-task-and-agent-run-links", !!t && hasLink(o, "issues", t) && !!worker && !!first && !!later && !!runHref(o, worker.id, first.id) && !!runHref(o, worker.id, later.id), "Each relevant execution links to the actual Morgan run detail and the originating website task."); - const runPart = (id: string) => o.reply.split(/\n/).filter(part => part.includes(id)).join(" "); - c.add("interrupted-and-continuation-states-explained", !!first && !!later && /\b(?:cancelled|canceled|interrupted|stopped)\b/i.test(runPart(first.id)) - && /\b(?:succeeded|successful|completed|finished)\b/i.test(runPart(later.id)) && /\bMorgan\b/.test(o.reply), - "Run-attributed reply rows distinguish the cancelled attempt from the successful continuation and identify the actual executing agent."); + // Which state words or agent naming the reply uses is not matched: the exact + // native run links (opened in verifyResult against each run's real status) + // and the run's own product read of the task's execution history are graded. + c.add("read-actual-task-run-history", o.runHistoryRead === true, + "The chat run read the website task's execution history through a product tool before answering."); c.add("no-history-rewrite-or-extra-owner", o.currentTask.id === t?.id && o.currentTask.assigneeAgentId === worker?.id && o.currentTask.status === "done" && history.every(run => [first?.id, later?.id].includes(run.id)) && !o.activity.some(row => row.entityId === t?.id && row.agentId === o.leadId && /issue\.(?:updated|deleted)/.test(row.action ?? "")), @@ -67,16 +84,22 @@ function findRuns(): ChatCatalogCase { await ctx.input.evidence("chat-catalog-real-interrupted-continuation.json", state); return state; } catch (error) { await gate.close(); gate = undefined; throw error; } }, gradeRunHistory); - return { ...base, expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660_000, + return { ...base, oracleVersion: "paperclip.chat-catalog.run-history.v2", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 } }, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-run-history.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts"], + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-run-history.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts"], async observe(ctx, state): Promise { if (!key) throw new Error("Run read probe was not created during setup"); const o = await observeInternal(ctx, state); const agentRunReadStatuses = await Promise.all(["interruptedRun", "continuationRun"].map(async name => { const url = new URL(`/api/heartbeat-runs/${entity(state, name).id}`, ctx.input.api.baseURL); const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key!.token}` } }); await response.body?.cancel(); return response.status; })); - return { ...o, agentRunReadStatuses, currentTask: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}`) }; + const taskId = entity(state, "task").id ?? "", runIds = ["interruptedRun", "continuationRun"].map(name => entity(state, name).id ?? ""); + let runHistoryRead = false; + for (const run of o.runs.filter(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id)) { + const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); + runHistoryRead = runHistoryRead || readTaskRunHistory(events, taskId, runIds); + } + return { ...o, agentRunReadStatuses, runHistoryRead, currentTask: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}`) }; }, async verifyResult(ctx, state, value) { const o = value as RunHistory; await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); const worker = entity(state, "worker"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 7a51d29bebd..989f2df63fa 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -72,7 +72,7 @@ function valid(scenario: string): InternalObservation { const first = { id: "interrupted", companyId: "company", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "task" } }; const later = { id: "continuation", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" } }; Object.assign(o.state, { worker: { id: "morgan" }, interruptedRun: first, continuationRun: later, interruptedTask: { ...o.tasks[0], executionRunId: "interrupted" }, gateCalls: [{ authorized: true, operation: "read_website_brief" }] }); - Object.assign(o, { currentTask: o.tasks[0], agentRunReadStatuses: [200, 200] }); o.runs.push(first, later); o.links.push("/ACORN/agents/morgan/runs/interrupted", "/ACORN/agents/morgan/runs/continuation"); + Object.assign(o, { currentTask: o.tasks[0], agentRunReadStatuses: [200, 200], runHistoryRead: true }); o.runs.push(first, later); o.links.push("/ACORN/agents/morgan/runs/interrupted", "/ACORN/agents/morgan/runs/continuation"); o.reply = "Morgan interrupted was cancelled.\nMorgan continuation succeeded."; break; } case "activity.agent-feedback": {