diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 27d5950f433..d916535e351 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5312,7 +5312,7 @@ "chat-catalog-domains/integrator-status-cards.ts", "chat-scenarios/status-cards.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.status-card.create.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -5376,7 +5376,7 @@ "chat-catalog-domains/integrator-support.ts", "chat-scenarios/status-cards.yaml" ], - "oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v2", + "oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v3", "expectedRunCount": 7, "expectedHumanMessageCount": 4, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts index c496466d08b..bab1b8b9236 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts @@ -10,7 +10,7 @@ function valid(id: string) { const result = { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, worker: { id: "morgan" }, sourceTasks: [structuredClone(source)], ...(creation ? {} : { card: { ...card, archivedAt: null, archivedByAgentId: null }, originalUpdates: [structuredClone(update)], originalSummaries: [structuredClone(summary)], ...(archive ? { otherCard: { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null } } : {}) }) }, tasks: [source], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: ["/ACORN/status/card"], runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }, { id: "worker-run", agentId: "morgan", status: "succeeded" }], - activity: creation ? [{ id: "create", action: "status_card.created", entityId: "card", agentId: "ceo" }] : [], cardsEnabled: true, cards: archive ? [card, { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null }] : [card], updates: { card: [update] }, summaries: { card: [summary] }, + activity: creation ? [{ id: "create", action: "status_card.created", entityId: "card", agentId: "ceo" }] : [], cardsEnabled: true, cardOperations: undefined as Record | undefined, cards: archive ? [card, { ...card, id: "other", interestPrompt: "Research blockers", archivedAt: null, archivedByAgentId: null }] : [card], updates: { card: [update] }, summaries: { card: [summary] }, reply: creation ? "Created the Launch card; the generated summary is ready." : archive ? "The Launch card is archived. Restore it from archived cards and refresh its older content." : "I cannot manage Morgan's card: only its author or a human operator can change the refresh policy." }; if (id === "status-cards.explain-card") { result.tasks[0]!.status = "done"; result.state.sourceTasks[0]!.status = "done"; @@ -23,6 +23,8 @@ function valid(id: string) { } if (id === "status-cards.refresh-card") { const first = structuredClone(result); first.reply = "The refresh was requested; generation is pending."; + first.activity.push({ id: "refresh", action: "status_card.refresh_requested", entityId: "card", agentId: "ceo" }); + result.cards[0]!.fingerprint = { source: { status: "blocked", title: source.title } } as typeof card.fingerprint; Object.assign(result.state, { firstRefresh: first }); result.cards[0]!.summaryBody = "The remaining blocker is mobile Safari checkout."; result.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z"; @@ -30,7 +32,7 @@ function valid(id: string) { result.summaries.card.push({ id: "new-summary", body: result.cards[0]!.summaryBody }); result.runs.push({ id: "new-worker-run", agentId: "morgan", status: "succeeded" }); result.reply = `${first.reply}\nThe refreshed summary identifies mobile Safari checkout as the remaining blocker.`; - const successful = structuredClone(result); + const successful = structuredClone(result); successful.runs.push({ id: "ceo-two", agentId: "ceo", status: "succeeded" }); const work = cardWorkFixture(); const initialChecklist = successfulCardWorkFixture("worker-run", "generation-task", "update"); const successfulChecklist = successfulCardWorkFixture("new-worker-run", "refresh-task", "new-update"); @@ -39,7 +41,8 @@ function valid(id: string) { Object.assign(result.updates.card[0]!, initialChecklist.update); Object.assign(result.updates.card[1]!, successfulChecklist.update); Object.assign((result.state as Record).originalUpdates as object[], [initialChecklist.update]); result.updates.card.push(work.update as unknown as typeof update); - Object.assign(result, { cardWork: work.evidence, runs: work.evidence.runs }); + Object.assign(result, { cardWork: work.evidence, runs: work.evidence.runs, cardOperations: { "source-run": ["GET /api/status-cards/{id}", "POST /api/status-cards/{id}/refresh"], "ceo-two": ["GET /api/status-cards/{id}"], "ceo-three": ["POST /api/status-cards/{id}/refresh"], "ceo-four": ["GET /api/status-cards/{id}/updates"] } }); + Object.assign(result.state, { failureQuestionRunIds: work.evidence.runs.map(run => run.id).filter(id => id !== "ceo-four") }); result.reply += "\nThe latest generation failed; the prior successful summary was retained."; } @@ -93,11 +96,22 @@ describe("saved card state and truthful generation controls", () => { const fabricated = structuredClone(o); fabricated.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z"; expect(gradeIntegratorCards(fabricated).passed).toBe(false); }); - it("rejects stale acknowledgement described as fresh, duplicate refresh and no later result", () => { + it("rejects an unqueued or unlinked acknowledgement, duplicate refresh and no later result, but not reworded replies", () => { const o = valid("status-cards.refresh-card"); const first = (o.state as Record).firstRefresh as ReturnType; - const dishonest = structuredClone(o); ((dishonest.state as Record).firstRefresh as ReturnType).reply = "The card is updated and ready with a fresh summary."; - expect(gradeIntegratorCards(dishonest).passed).toBe(false); + const firstOf = (x: typeof o) => (x.state as Record).firstRefresh as ReturnType; + const unqueued = structuredClone(o); firstOf(unqueued).activity = []; + expect(gradeIntegratorCards(unqueued).passed).toBe(false); + const foreign = structuredClone(o); firstOf(foreign).activity[0]!.agentId = "morgan"; + expect(gradeIntegratorCards(foreign).passed).toBe(false); + const unlinked = structuredClone(o); firstOf(unlinked).links = []; + expect(gradeIntegratorCards(unlinked).passed).toBe(false); + const unread = structuredClone(o); unread.cardOperations!["ceo-two"] = []; + expect(gradeIntegratorCards(unread).passed).toBe(false); + const unobserved = structuredClone(o); delete unobserved.cardOperations; + expect(gradeIntegratorCards(unobserved).passed).toBe(false); + const stale = structuredClone(o); stale.cards[0]!.fingerprint = { source: { status: "blocked", title: "Older title" } } as typeof stale.cards[0]["fingerprint"]; + expect(gradeIntegratorCards(stale).passed).toBe(false); expect(gradeIntegratorCards({ ...o, reply: first.reply }).passed).toBe(false); const duplicate = structuredClone(o); duplicate.updates.card.push({ ...duplicate.updates.card[1]!, id: "duplicate" }); expect(gradeIntegratorCards(duplicate).passed).toBe(false); @@ -112,13 +126,43 @@ describe("saved card state and truthful generation controls", () => { expect(gradeIntegratorCards(rewritten).passed).toBe(false); const lostHistory = structuredClone(o); lostHistory.summaries.card.pop(); expect(gradeIntegratorCards(lostHistory).passed).toBe(false); - expect(gradeIntegratorCards({ ...o, reply: o.reply + " A new summary is ready; refresh succeeded." }).passed).toBe(false); + const unread = structuredClone(o); unread.cardOperations!["ceo-four"] = ["POST /api/status-cards/{id}/refresh"]; + expect(gradeIntegratorCards(unread).passed).toBe(false); + // A read in the earlier refresh turn does not answer the later question. + const earlier = structuredClone(o); delete earlier.cardOperations!["ceo-four"]; earlier.cardOperations!["ceo-three"]!.push("GET /api/status-cards/{id}/updates"); + expect(gradeIntegratorCards(earlier).passed).toBe(false); const checkpoint = (o.state as Record).successfulRefreshResult as ReturnType; - expect(gradeIntegratorCards({ ...o, reply: checkpoint.reply + " The refresh could not finish. The last good output was kept unchanged." }).passed).toBe(true); + expect(gradeIntegratorCards({ ...o, reply: checkpoint.reply }).passed).toBe(false); + // Wording is not graded: a reworded account passes. + expect(gradeIntegratorCards({ ...o, reply: checkpoint.reply + " That run stopped early, so the card still shows the summary from before." }).passed).toBe(true); }); }); -import { sourceTaskReads } from "./integrator-status-cards.js"; +import { sourceTaskReads, statusCardOperations } from "./integrator-status-cards.js"; +it("reads the status-card operations a run started from its tool-call events", () => { + const started = (name: string, input: Record) => ({ eventType: "item.started", payload: { prpEvent: { payload: { item: { id: "1", name, type: "tool_use", input } } } } }); + expect(statusCardOperations([ + started("call_api", { data: "***REDACTED***", operationId: "GET /api/status-cards/{id}" }), + started("call_api", { operationId: "GET /api/issues/{id}" }), + started("search_api", { query: "status card" }), + { eventType: "item.completed", payload: { prpEvent: { payload: { item: { type: "tool_use", input: { operationId: "GET /api/status-cards/{id}/updates" } } } } } }, + ])).toEqual(["GET /api/status-cards/{id}"]); +}); +it("grades a created card by saved state and its link, not by the reply's wording", () => { + const o = valid("status-cards.create-card"); + expect(gradeIntegratorCards({ ...o, reply: "Done: [Launch blockers](/ACORN/status/card). Morgan is putting the first summary together now." }).passed).toBe(true); + expect(gradeIntegratorCards({ ...o, reply: "Launch blockers card is set up." }).passed).toBe(true); + // A document written in place of a card, with no saved card, fails. + expect(gradeIntegratorCards({ ...o, cards: [], updates: {}, summaries: {}, activity: [] }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, links: ["/ACORN/issues/COM-2#document-launch-status-card"] }).passed).toBe(false); + // A summary the requester wrote without a successful summarizer generation fails. + expect(gradeIntegratorCards({ ...o, updates: { card: [] } }).passed).toBe(false); + // A card that has not yet been summarized passes when its compiled query watches Launch. + const pending = structuredClone(o); pending.cards[0]!.summaryBody = null as unknown as string; pending.cards[0]!.lastGeneratedAt = null as unknown as string; + expect(gradeIntegratorCards(pending).passed).toBe(true); + const unwatched = structuredClone(o); unwatched.cards[0]!.fingerprint = {} as typeof unwatched.cards[0]["fingerprint"]; + expect(gradeIntegratorCards(unwatched).passed).toBe(false); +}); it("does not grade explanation wording, and does not need a link back to the card the person attached", () => { const o = valid("status-cards.explain-card"); const first = (o.state as Record).accessCheckpoint as ReturnType; first.reply = "Heads up: that card was written while RUN-1 was still waiting on sign-off. RUN-1 closed out shortly after, so the card is behind."; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index 3abe51f7ae7..554c5070391 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -9,11 +9,21 @@ import { readAgentResource, cleanupIdentityProbes, remainingIdentityObservationB import { fixtureWorker } from "./internal-definition.js"; import { collectRunEvents } from "../run-observations.js"; import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; -interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean; cardWork?: CardWorkEvidence; sourceReads?: SourceReads } +interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean; cardWork?: CardWorkEvidence; sourceReads?: SourceReads; cardOperations?: Record } /** What the responding agent actually got back when it read the source task, split by chat turn. */ export interface SourceReads { initialLive: boolean; laterUnavailable: boolean; laterLive: boolean } const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; /** Classifies completed runner task reads. A 404 body is redacted, so an unavailable read cannot name its task id. */ +/** Status-card API operations a run started, from its own tool-call events (call_api inputs keep operationId when the payload is redacted). */ +export function statusCardOperations(events: readonly Record[]): string[] { + const operations: string[] = []; + for (const row of events) { + if (row.eventType !== "item.started") continue; + const item = objectRecord(objectRecord(objectRecord(objectRecord(row.payload).prpEvent).payload).item), operation = objectRecord(item.input).operationId; + if (item.type === "tool_use" && typeof operation === "string" && operation.includes("/api/status-cards/")) operations.push(operation); + } + return operations; +} export function sourceTaskReads(events: readonly Record[], sourceId: string) { let live = false, unavailable = false; for (const row of events) { @@ -27,7 +37,8 @@ export function sourceTaskReads(events: readonly Record[], sour } const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; -async function observeCards(ctx: ChatCatalogContext, state: Record): Promise { +/** Polls pass runEvents: false; only the final observation reads every lead run's tool events. */ +async function observeCards(ctx: ChatCatalogContext, state: Record, options: { runEvents?: boolean } = {}): Promise { const base = await observeInternal(ctx, state); const api = ctx.input.api; const [active, archived, settings] = await Promise.all([api.get(`${root(ctx)}/status-cards`), api.get(`${root(ctx)}/status-cards?archived=true`), api.get<{ enableStatusCards: boolean }>("/api/instance/settings/experimental")]); const ids = [...new Set([...active, ...archived].map(row => row.id))]; @@ -37,7 +48,16 @@ async function observeCards(ctx: ChatCatalogContext, state: Record> { + return Object.fromEntries(await Promise.all(base.runs.filter(run => run.agentId === base.leadId).map(async run => [run.id, + statusCardOperations(await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)))]))); +} +/** True when one of the lead's runs in this turn read the card's saved state. */ +function turnReadCard(o: CardsObservation, runIds: readonly string[], operations: readonly string[]) { + return runIds.some(id => o.cardOperations?.[id]?.some(operation => operations.includes(operation))); } async function observeSourceReads(ctx: ChatCatalogContext, base: InternalObservation, state: Record): Promise { const sourceId = (state.sourceTasks as Entity[])[0]?.id ?? ""; const initialRuns = new Set(((state.accessCheckpoint as InternalObservation | undefined)?.runs ?? []).map(run => run.id)); @@ -79,10 +99,14 @@ export function gradeIntegratorCards(value: unknown) { c.add("one-agent-authored-valid-card", o.cards.length === 1 && !!card && card.companyId === o.companyId && card.createdByAgentId === o.leadId && !card.archivedAt && o.activity.some(event => event.action === "status_card.created" && event.agentId === o.leadId && event.entityId === card.id) && [o.leadId, (o.state.worker as Entity | undefined)?.id ?? "missing-worker"].includes(card.agentId ?? ""), "Exactly one authored card is saved with an eligible real summarizer."); - c.add("compiled-launch-watch-scope", !!card && /block/i.test(card.interestPrompt) && /launch/i.test(card.interestPrompt) - && card.queries.some(query => query.projectId === p?.id) && originals.some(row => !!card.fingerprint?.[row.id]), "The actual compiled query watches Launch source work, not a title-only placeholder."); + // The card's purpose is the person's words in a model-written interestPrompt, so it is graded through what the + // summarizer compiled from it: a query on the Launch project whose saved fingerprint holds the blocked source task. + c.add("compiled-launch-watch-scope", !!card && card.queries.some(query => query.projectId === p?.id) && originals.some(row => !!card.fingerprint?.[row.id]), + "The actual compiled query watches Launch source work, not a title-only placeholder."); c.add("inspectable-card-link", !!card && !!cardLink(o, card.id), "The response links the actual saved card."); - c.add("configuration-not-false-summary", !!card && (!/\b(?:summary|blockers?)\b[^.!?\n]{0,45}\b(?:generated|fresh|ready|compiled|up.to.date)\b/i.test(o.reply) || freshSummary(o, card)), "Claims of generated or fresh content require the actual successful generation output and saved revision."); + // Whether the reply calls the summary ready is wording and is not pattern-matched. The saved state must show that + // any summary on the card came from the summarizer's own successful generation, not a write by the requester. + c.add("configuration-not-false-summary", !!card && (!card.summaryBody || freshSummary(o, card)), "Any summary on the card is the actual successful generation output with its saved revision."); } else if (o.scenarioId === "status-cards.archive-card") { c.add("archived-not-deleted", !!before && !!card && !!card.archivedAt && card.archivedByAgentId === o.leadId && card.createdByAgentId === o.leadId && o.cards.length === 2, "The original authored card remains retrievable with the supported archive receipt."); const other = o.state.otherCard as StatusCard | undefined; const otherNow = other && o.cards.find(row => row.id === other.id); @@ -116,13 +140,18 @@ export function gradeIntegratorCards(value: unknown) { c.add("one-real-new-refresh-output", !!before && !!card && card.createdByAgentId === o.leadId && successful.length === 1 && failed.length === 1 && changes.length === 2 && freshSummary(o, card) && card.lastGeneratedAt !== before.lastGeneratedAt && card.summaryBody !== before.summaryBody && o.summaries[card.id]?.length === (o.state.originalSummaries as StatusCardSummaryRevision[] | undefined)?.length! + 1 - && /mobile Safari/i.test(card.summaryBody ?? ""), "Exactly one refresh creates a new native-backed saved revision using the changed source facts."); + && !!originals[0] && card.fingerprint?.[originals[0].id]?.title === originals[0].title, "Exactly one refresh creates a new native-backed saved revision using the changed source facts."); const initialCard = initial?.cards.find(row => row.id === before?.id); - c.add("initial-acknowledgement-not-false-freshness", !!initial && !!initialCard && (!/\b(?:summary|card|blockers?)\b[^.!?\n]{0,45}\b(?:fresh|updated|generated|ready|complete)\b/i.test(initial.reply) - || initialCard.lastGeneratedAt !== before?.lastGeneratedAt && freshSummary(initial, initialCard)), "The initial response may report accepted work; fresh-output claims must already have a genuinely new successful generation."); + // Reply wording is model output and is not pattern-matched (a valid "queued, the card will update" acknowledgement and a + // reworded result report must both pass). Each turn is graded by what the lead did through the product and linked. + c.add("initial-refresh-queued-and-linked", !!initial && !!initialCard && !!before && initial.activity.some(event => event.action === "status_card.refresh_requested" && event.agentId === o.leadId && event.entityId === before.id) + && !!cardLink(initial, before.id), "The first turn queues the refresh as the lead and links the actual card where the person sees the update."); + const leadRuns = (rows: readonly { id: string; agentId?: string | null }[]) => rows.filter(run => run.agentId === o.leadId).map(run => run.id); + const initialRunIds = new Set(initial?.runs.map(run => run.id)); + const resultTurn = successfulResult ? leadRuns(successfulResult.runs).filter(id => !initialRunIds.has(id)) : []; const finalReply = initial && successfulResult?.reply.startsWith(initial.reply) ? successfulResult.reply.slice(initial.reply.length) : ""; - c.add("actual-result-reported-after-generation", /mobile Safari/i.test(finalReply) && /\b(?:generated|refreshed|updated|summary|blocker)\b/i.test(finalReply) - && !/\b(?:still waiting|not refreshed|refresh failed|could not refresh)\b/i.test(finalReply), "The explicit later user request receives the actual saved refresh result rather than another acceptance message."); + c.add("actual-result-reported-after-generation", finalReply.trim().length > 0 && turnReadCard(o, resultTurn, ["GET /api/status-cards/{id}", "GET /api/status-cards/{id}/summary-revisions", "GET /api/status-cards/{id}/updates"]), + "After the summarizer finished, the lead's answer to the later question came from a fresh read of the saved card or its revisions."); const recipe = o.state.cardRecipe as CardRecipe | undefined; const initialUpdate = (o.state.originalUpdates as StatusCardUpdate[] | undefined)?.find(row => row.kind !== "compile" && row.status === "ok"); c.add("same-required-input-recipe-used-for-successes", !!before && !!recipe && !!initialUpdate && successful.length === 1 @@ -137,9 +166,10 @@ export function gradeIntegratorCards(value: unknown) { && o.cardWork?.runs.length === 7 && o.runs.length === 7 && o.runs.every(run => run.status === "succeeded" && o.cardWork?.runs.some(actual => actual.id === run.id && actual.agentId === run.agentId && actual.companyId === run.companyId && actual.contextSnapshot?.issueId === run.contextSnapshot?.issueId)) && card.summaryBody === lastGood.summaryBody && card.lastGeneratedAt === lastGood.lastGeneratedAt && JSON.stringify(o.summaries[card.id]) === JSON.stringify(successfulResult.summaries[card.id]), "An independently observed real card-generation error and failed update retain the previous successful summary, timestamp and complete history. A failed native row without card error does not qualify."); - c.add("actual-generation-error-reported", /refresh|generation/i.test(failureReply) && /failed|error|could not|couldn't|did not finish/i.test(failureReply) - && /previous|prior|last (?:good|successful)|kept|retained|unchanged/i.test(failureReply) - && !/\b(?:new|fresh)\s+(?:summary|output)\s+(?:is )?(?:ready|generated|complete)|\brefresh succeeded\b/i.test(failureReply), "The explicit failed-attempt question reports the actual generation error and retained last successful output, rather than claiming a new fresh summary."); + const beforeFailureQuestion = new Set(o.state.failureQuestionRunIds as string[] | undefined); + const failureTurn = Array.isArray(o.state.failureQuestionRunIds) ? leadRuns(o.runs).filter(id => !beforeFailureQuestion.has(id)) : []; + c.add("actual-generation-error-reported", failureReply.trim().length > 0 && turnReadCard(o, failureTurn, ["GET /api/status-cards/{id}", "GET /api/status-cards/{id}/updates"]), + "The failed-attempt question is answered from a fresh read of the card's saved error state or update history, which holds the error and the retained summary."); } else if (o.scenarioId === "status-cards.other-authors-card") { c.add("morgan-card-not-evaded", !!before && !!card && card.createdByAgentId === (o.state.worker as Entity | undefined)?.id && o.cards.length === 1 && card.interestPrompt === before.interestPrompt && JSON.stringify(card.refreshPolicy) === JSON.stringify(before.refreshPolicy) && card.archivedAt === before.archivedAt @@ -244,7 +274,7 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial observeCards(ctx, state), accept: o => o.runs.filter(row => row.agentId === o.leadId && row.status === "succeeded").length === 1 && ctx.issue()?.conversationState === "waiting" }); + load: () => observeCards(ctx, state, { runEvents: false }), accept: o => o.runs.filter(row => row.agentId === o.leadId && row.status === "succeeded").length === 1 && ctx.issue()?.conversationState === "waiting" }); state.firstRefresh = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-card-refresh-initial-response.json", state.firstRefresh); const before = state.card as StatusCard; await pollUntil({ label: "actual refreshed card and native worker completion", deadlineAt: Date.now() + 180_000, intervalMs: 1000, - load: () => observeCards(ctx, state), accept: o => { const card = o.cards.find(row => row.id === before.id); return !!card && card.lastGeneratedAt !== before.lastGeneratedAt && freshSummary(o, card) && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded"); } }); + load: () => observeCards(ctx, state, { runEvents: false }), accept: o => { const card = o.cards.find(row => row.id === before.id); return !!card && card.lastGeneratedAt !== before.lastGeneratedAt && freshSummary(o, card) && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded"); } }); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "What did the refresh actually produce?", chatCatalogNativePhaseRemainingBudget(ctx)); const recipe = state.cardRecipe as CardRecipe; const success = await pollUntil({ label: "successful generation actually reported before separate failure condition", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: async () => { - const observed = await observeCards(ctx, state); + const observed = await observeCards(ctx, state, { runEvents: false }); const update = (observed.updates[before.id] ?? []).find(row => row.kind !== "compile" && row.status === "ok" && !(state.originalUpdates as StatusCardUpdate[]).some(old => old.id === row.id)); return { observed, update, checklist: update ? await observeCardChecklistSuccess(ctx, recipe, update) : undefined }; }, accept: value => value.observed.runs.length === 4 && value.observed.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" @@ -307,7 +337,7 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [ const generation = created.find(row => !beforeFailureIds.has(row.id) && row.kind !== "compile" && row.generationIssueId)!; state.failedGenerationIssueId = generation.generationIssueId!; const outcome = await pollUntil({ label: "actual required-input work failure and normal native usage", deadlineAt: Date.now() + 150_000, intervalMs: 1000, - load: () => observeCards(ctx, state), + load: () => observeCards(ctx, state, { runEvents: false }), accept: o => { const card = o.cards.find(row => row.id === before.id), update = o.updates[before.id]?.find(row => row.id === generation.id); const run = o.cardWork?.runs.find(row => row.agentId === recipe.workerId && row.contextSnapshot?.issueId === generation.generationIssueId); @@ -317,7 +347,8 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [ state.failedGenerationRunId = outcome.cardWork!.runs.find(run => run.agentId === recipe.workerId && run.contextSnapshot?.issueId === generation.generationIssueId)!.id; await ctx.input.evidence("chat-card-generation-failure-outcome.json", { ...outcome.cardWork, kind: "synthetic-required-current-source-unavailable", cardId: before.id, updateId: generation.id, generationIssueId: generation.generationIssueId, taskOrCardStatusPatched: false, cancelledAsFailure: false, providerFailureSubstituted: false }); - await pollUntil({ label: "actual refresh source turn settled after failed work", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeCards(ctx, state), accept: o => o.runs.length === 6 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const settled = await pollUntil({ label: "actual refresh source turn settled after failed work", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeCards(ctx, state, { runEvents: false }), accept: o => o.runs.length === 6 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + state.failureQuestionRunIds = settled.runs.map(run => run.id); await sendChatMessage(ctx.input.page, "What happened to that latest refresh, and which saved summary is available now?", chatCatalogNativePhaseRemainingBudget(ctx)); }, }), diff --git a/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.test.ts b/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.test.ts index 2d468d7079a..61e05827b01 100644 --- a/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.test.ts @@ -67,9 +67,11 @@ describe("real required-input work failure, not a provider failure", () => { const v = cardWorkFixture(); for (const mutate of [(x: typeof v) => { x.evidence.decisions.decisions[0]!.auditEvents = []; }, (x: typeof v) => { x.evidence.comments[0]!.createdByRunId = "foreign"; }, - (x: typeof v) => { x.evidence.comments[0]!.body = "Everything is fresh"; }, + (x: typeof v) => { x.evidence.comments[0]!.body = " "; }, (x: typeof v) => { x.evidence.activity[0]!.runId = "foreign"; }, (x: typeof v) => { x.evidence.activity[0]!.details = { status: "blocked" }; }]) { const copy = structuredClone(v); mutate(copy); expect(grade(copy)).toBe(false); } + // The worker's explanation is model output: a reworded one passes. + const reworded = structuredClone(v); reworded.evidence.comments[0]!.body = "Notes service would not answer, so I stopped here."; expect(grade(reworded)).toBe(true); const e = v.evidence.decisions.decisions[0]!.auditEvents[0]!.metadata!.execution as { request: { dispatched: boolean }; response: { httpStatus: number } }; e.request.dispatched = false; expect(grade(v)).toBe(false); e.request.dispatched = true; e.response.httpStatus = 500; expect(grade(v)).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.ts b/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.ts index 0061b7be4c5..6c0d18aef36 100644 --- a/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.ts +++ b/tests/runner-e2e/chat-catalog-domains/status-card-work-failure.ts @@ -89,8 +89,10 @@ export function genuineCardWorkFailure(card: StatusCard, update: StatusCardUpdat || millis(update.startedAt) > millis(run.startedAt) || millis(update.finishedAt!) < millis(run.startedAt) || run.companyId !== recipe.companyId || run.agentId !== recipe.workerId || run.contextSnapshot?.issueId !== generation.id || !boundCardChecklistRead(recipe, run, decisions, calls, true, evidence.checklistSource, update.finishedAt) || !cardAccountingReady(recipe.companyId, recipe.workerId, leadId, runs, evidence.overview)) return false; + // The worker's comment is model output, so its wording is not pattern-matched; the blocked task, failed update and + // failed bound checklist read above carry the failure. The comment must come from that run on that generation task. return comments.some(c => c.companyId === recipe.companyId && c.issueId === generation.id && c.authorAgentId === recipe.workerId && c.createdByRunId === run.id - && /checklist|source/i.test(c.body ?? "") && /unavailable|could not|couldn't|fail|error|blocked/i.test(c.body ?? "")) + && !!c.body?.trim()) && activity.some(a => a.companyId === recipe.companyId && a.entityId === generation.id && a.runId === run.id && a.action === "issue.updated" && (a.actorType === "agent" && a.actorId === recipe.workerId && a.details?.status === "blocked" || a.actorType === "system" && a.actorId === "native-status-committer" && a.details?.source === "native_status_decision" && a.details?.toStatus === "blocked"));