From d75c7c548fbb2075a673cb6fdddcd013867f1166 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 09:44:35 -0600 Subject: [PATCH] test(evals): grade explain-card by the agent's source reads, not reply wording The explain-card grader required regex matches on the model's explanation and a link back to the status card the person had attached. A correct, grounded reply failed. Grade what the agent actually did instead: it read the live source task and linked it before explaining, and after the source was deleted its follow-up turn attempted a fresh read that returned 404 with no successful read. Explanation wording is no longer graded; a reply that misstates a correctly read live status is not detected by this oracle. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 3 +- .../integrator-status-cards.test.ts | 43 +++++++++------ .../integrator-status-cards.ts | 54 +++++++++++++------ 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 38de957787a..a0193691696 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5319,9 +5319,10 @@ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", + "run-observations.ts", "chat-scenarios/status-cards.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.status-card.explain.v2", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts index 9c106279739..220d7cec1db 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts @@ -17,6 +17,7 @@ function valid(id: string) { result.reply = "The card reflects its older observation: payment approval blocked Launch. The source task is now done and the payment approval is resolved."; result.links.push("/ACORN/issues/source"); const first = structuredClone(result); Object.assign(result.state, { accessCheckpoint: first, sourceReadStatus: 404, sourceUnavailable: true, accessPrincipalId: "ceo" }); + Object.assign(result, { sourceReads: { initialLive: true, laterUnavailable: true, laterLive: false } }); result.tasks = []; result.reply = `${first.reply}\nI can no longer read or verify the current source details; the task is unavailable. The older saved card is historical context only.`; } @@ -72,11 +73,16 @@ describe("saved card state and truthful generation controls", () => { const stolen = structuredClone(o); stolen.cards[0]!.createdByAgentId = "ceo"; expect(gradeIntegratorCards(stolen).passed).toBe(false); }); - it("rejects an older blocker presented as current and missing source grounding", () => { - const o = valid("status-cards.explain-card"); - expect(gradeIntegratorCards({ ...o, reply: "The card is current and up-to-date. Payment approval still blocks Launch although the task is done." }).passed).toBe(false); - expect(gradeIntegratorCards({ ...o, links: ["/ACORN/status/card"] }).passed).toBe(false); - const falseCurrent = structuredClone(o); falseCurrent.reply += " I verified the current restricted task."; expect(gradeIntegratorCards(falseCurrent).passed).toBe(false); + it("requires the live source read, its link and the later unavailable read", () => { + const o = valid("status-cards.explain-card") as ReturnType & { sourceReads: { initialLive: boolean; laterUnavailable: boolean; laterLive: boolean } }; + expect(gradeIntegratorCards(o).passed).toBe(true); + expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, initialLive: false } }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, laterUnavailable: false } }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, laterLive: true } }).passed).toBe(false); + const { sourceReads: _omitted, ...unobserved } = o; expect(gradeIntegratorCards(unobserved).passed).toBe(false); + const unlinked = structuredClone(o); ((unlinked.state as Record).accessCheckpoint as ReturnType).links = ["/ACORN/status/card"]; + expect(gradeIntegratorCards(unlinked).passed).toBe(false); + const silent = structuredClone(o); silent.reply = ((silent.state as Record).accessCheckpoint as ReturnType).reply; expect(gradeIntegratorCards(silent).passed).toBe(false); const unrelated = structuredClone(o); Object.assign(unrelated.state, { accessPrincipalId: "someone-else" }); expect(gradeIntegratorCards(unrelated).passed).toBe(false); const fabricated = structuredClone(o); fabricated.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z"; expect(gradeIntegratorCards(fabricated).passed).toBe(false); @@ -106,19 +112,24 @@ describe("saved card state and truthful generation controls", () => { }); }); -import { explainsSavedStaleness } from "./integrator-status-cards.js"; -it("accepts natural saved-wording/live-status distinction without requiring the literal word card", () => { - expect(explainsSavedStaleness("It's stale wording, not a current blocker. RUN-1 is done and says the payment approval was received.")).toBe(true); - expect(explainsSavedStaleness("The wording is not stale. Payment approval is completed.")).toBe(false); - expect(explainsSavedStaleness("The summary is fresh. Payment approval is resolved.")).toBe(false); - expect(explainsSavedStaleness("The older summary is stale, but Launch is still blocked. Payment approval is completed.")).toBe(false); -}); -it("a paraphrase cannot replace required saved-card link or source-access evidence", () => { +import { sourceTaskReads } from "./integrator-status-cards.js"; +it("does not grade explanation wording, and does not need a link back to the card the person attached", () => { const o = valid("status-cards.explain-card"); const first = (o.state as Record).accessCheckpoint as ReturnType; - first.reply = "It's stale wording, not a current blocker. The source task is done and the payment approval was received."; - o.reply = first.reply + "\nI can no longer read or verify its current state; only the older saved summary remains."; + first.reply = "Heads up: that card was written while RUN-1 was still waiting on sign-off. RUN-1 closed out shortly after, so the card is behind."; + first.links = ["/ACORN/issues/source"]; + o.reply = first.reply + "\nThe task is gone now (404), so all I have is the old snapshot."; + o.links = ["/ACORN/issues/source"]; expect(gradeIntegratorCards(o).passed).toBe(true); - expect(gradeIntegratorCards({ ...o, links: ["/ACORN/issues/source"] }).passed).toBe(false); +}); +it("classifies live and unavailable task reads from runner tool output", () => { + const event = (output: string, status = "completed") => ({ eventType: "tool.execution.completed", payload: { prpEvent: { sourceKind: "runner", payload: { name: "paperclip_get_task", status, output } } } }); + const live = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", data: { id: "source", status: "done" } }) })); + const gone = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", ok: false, status: 404, data: "***REDACTED***" }) })); + const other = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", data: { id: "other" } }) })); + expect(sourceTaskReads([live], "source")).toEqual({ live: true, unavailable: false }); + expect(sourceTaskReads([gone], "source")).toEqual({ live: false, unavailable: true }); + expect(sourceTaskReads([other], "source")).toEqual({ live: false, unavailable: false }); + expect(sourceTaskReads([event(JSON.parse(JSON.stringify(live.payload.prpEvent.payload.output)), "running")], "source")).toEqual({ live: false, unavailable: false }); }); import type { StatusCard } from "../../../packages/shared/src/validators/status-card.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index f3a7070ebd9..b0a6a6d2032 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -7,8 +7,25 @@ import { checks, observation, project, task, sameTask, hasLink, type Entity, typ import { prepareCardChecklist, updateCardChecklistSource, observeCardChecklistSuccess, observeCardWork, failCardChecklist, genuineCardWorkFailure, genuineCardChecklistSuccess, cardAccountingReady, cleanupCardChecklist, type CardRecipe, type CardWorkEvidence } from "./status-card-work-failure.js"; import { readAgentResource, cleanupIdentityProbes, remainingIdentityObservationBudget } from "./identity-support.js"; import { fixtureWorker } from "./internal-definition.js"; +import { collectRunEvents } from "../run-observations.js"; import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; -interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean; cardWork?: CardWorkEvidence } +interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean; cardWork?: CardWorkEvidence; sourceReads?: SourceReads } +/** What the responding agent actually got back when it read the source task, split by chat turn. */ +export interface SourceReads { initialLive: boolean; laterUnavailable: boolean; laterLive: boolean } +const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +/** Classifies completed runner task reads. A 404 body is redacted, so an unavailable read cannot name its task id. */ +export function sourceTaskReads(events: readonly Record[], sourceId: string) { + let live = false, unavailable = false; + for (const row of events) { + const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload); + const output = typeof payload.output === "string" ? payload.output : ""; + if (row.eventType !== "tool.execution.completed" || envelope.sourceKind !== "runner" || payload.status !== "completed" || !output.includes("GET /api/issues/{id}")) continue; + if (/\\*"status\\*":404\b/.test(output)) unavailable = true; + else if (!/\\*"ok\\*":false/.test(output) && output.includes(sourceId)) live = true; + } + return { live, unavailable }; +} + const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; async function observeCards(ctx: ChatCatalogContext, state: Record): Promise { const base = await observeInternal(ctx, state); const api = ctx.input.api; @@ -19,7 +36,19 @@ async function observeCards(ctx: ChatCatalogContext, state: Record [id, await api.get(`/api/status-cards/${id}/summary-revisions`)]))); const cardWork = ctx.scenario.id === "status-cards.refresh-card" && state.cardRecipe && typeof state.failedGenerationIssueId === "string" ? await observeCardWork(ctx, state.cardRecipe as CardRecipe, state.failedGenerationIssueId) : undefined; - return { ...base, cards, updates, summaries, cardsEnabled: settings.enableStatusCards, ...(cardWork ? { cardWork } : {}) }; + const sourceReads = ctx.scenario.id === "status-cards.explain-card" && state.sourceUnavailable === true ? await observeSourceReads(ctx, base, state) : undefined; + return { ...base, cards, updates, summaries, cardsEnabled: settings.enableStatusCards, ...(cardWork ? { cardWork } : {}), ...(sourceReads ? { sourceReads } : {}) }; +} +async function observeSourceReads(ctx: ChatCatalogContext, base: InternalObservation, state: Record): Promise { + const sourceId = (state.sourceTasks as Entity[])[0]?.id ?? ""; const initialRuns = new Set(((state.accessCheckpoint as InternalObservation | undefined)?.runs ?? []).map(run => run.id)); + const reads: SourceReads = { initialLive: false, laterUnavailable: false, laterLive: false }; + for (const run of base.runs.filter(row => row.agentId === base.leadId)) { + const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); + const found = sourceTaskReads(events, sourceId); + if (initialRuns.has(run.id)) reads.initialLive ||= found.live; + else { reads.laterUnavailable ||= found.unavailable; reads.laterLive ||= found.live; } + } + return reads; } function cardsObservation(value: unknown): CardsObservation | undefined { const o = value as CardsObservation; @@ -31,11 +60,6 @@ function freshSummary(o: CardsObservation, card: StatusCard) { && o.runs.some(run => run.id === row.runId && run.status === "succeeded") && row.generationIssueId != null) && o.summaries[card.id]?.some(row => row.body === card.summaryBody); } -export function explainsSavedStaleness(reply: string) { - return groundedStatement(reply, /\b(?:card|summary|observation|wording)\b/i, /\b(?:stale|older|previous|last|outdated)\b/i, - /\b(?:still|currently|remains)\s+(?:is\s+)?blocked\b|\b(?:summary|card)\s+(?:is\s+)?(?:current|up.to.date|fresh)\b|\b(?:not|isn.t|is not)\s+(?:stale|older|outdated)\b/i) - && /payment approval/i.test(reply) && /\b(?:done|resolved|completed|approved)\b/i.test(reply); -} export function gradeIntegratorCards(value: unknown) { const c = checks(); const o = cardsObservation(value); c.add("public-card-generation-history", !!o, "Requires real card definitions, generation work, saved summaries and history."); @@ -66,13 +90,12 @@ export function gradeIntegratorCards(value: unknown) { const source = originals[0]; c.add("saved-observation-is-stale", !!before && !!card && !!source && card.summaryBody === before.summaryBody && card.lastGeneratedAt === before.lastGeneratedAt && card.fingerprint?.[source.id]?.status === "blocked" && source.status === "done" && freshSummary(o, card), "The real previous generated blocker summary is retained while the actual source task is now done."); - const initial = cardsObservation(o.state.accessCheckpoint); - c.add("saved-versus-live-explained", !!initial && explainsSavedStaleness(initial.reply), "The original canonical explanation grounds the blocker in its older generation and distinguishes the then-resolved accessible source; a later follow-up cannot repair it."); - const later = initial && o.reply.startsWith(initial.reply) ? o.reply.slice(initial.reply.length) : ""; + const initial = cardsObservation(o.state.accessCheckpoint); const later = initial && o.reply.startsWith(initial.reply) ? o.reply.slice(initial.reply.length).trim() : ""; + // The explanation's wording is model output, so it is not pattern-matched. These checks grade what the + // agent read and linked; a reply that misstates a correctly read live status is not detected here. + c.add("live-source-read-before-explaining", !!initial && !!source && o.sourceReads?.initialLive === true && hasLink(initial, "issues", source), "Before answering, the responding agent actually read the source task's live record and linked it for the person."); c.add("actual-current-source-access-loss", !!initial && o.state.sourceReadStatus === 404 && o.state.sourceUnavailable === true && o.state.accessPrincipalId === o.leadId - && /(?:cannot|can't|no longer|lack|missing|not (?:able|permitted)|lost)/i.test(later) && /access|read|permission|verif|check/i.test(later) - && !/\b(?:I (?:have )?(?:verified|read|checked)|currently verified)\b/i.test(later), "After an explicit later unavailable-source question, the actual responding principal's public read is404 after genuine deletion. Chat cannot newly ground current facts; legitimate earlier cached summary/history remains allowed. This is not permission-denial proof."); - c.add("saved-card-and-live-task-links", !!card && !!source && !!cardLink(o, card.id) && hasLink(o, "issues", source), "The explanation links the actual saved card and its changed source task."); + && o.sourceReads?.laterUnavailable === true && o.sourceReads.laterLive === false && later.length > 0, "After the source was deleted, the follow-up turn attempted a fresh read, received the actual404 and answered without any successful source read. Earlier cached summary/history remains allowed."); } else if (o.scenarioId === "status-cards.refresh-card") { const initial = cardsObservation(o.state.firstRefresh); const oldIds = new Set((o.state.originalUpdates as StatusCardUpdate[] | undefined)?.map(row => row.id)); const changes = card ? o.updates[card.id]?.filter(row => !oldIds.has(row.id) && row.kind !== "compile") ?? [] : []; @@ -209,12 +232,11 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [ define("status-cards.create-card", "none"), define("status-cards.archive-card", "ceo", { authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, attemptTimeoutMs: 720_000 }), define("status-cards.other-authors-card", "morgan"), - define("status-cards.explain-card", "ceo", { authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, - sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/status-cards.yaml"], + define("status-cards.explain-card", "ceo", { oracleVersion: "paperclip.chat-catalog.status-card.explain.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, + sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "run-observations.ts", "chat-scenarios/status-cards.yaml"], async act(ctx, state) { const first = await pollUntil({ label: "accessible source explanation before actual access change", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000, load: () => observeCards(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting") }); - if (!explainsSavedStaleness(first.reply)) throw new Error("Initial saved-versus-live explanation was incorrect; later unavailability cannot repair it"); state.accessCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-card-explanation-before-access-loss.json", state.accessCheckpoint); const source = (state.sourceTasks as Entity[])[0]!; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${source.identifier ?? source.id}`, source.title!, resultHref(first, source.identifier ?? source.id, "issues") ?? resultHref(first, source.id, "issues"));