diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index a1a5b56b3f3..2b07b5916a2 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5283,7 +5283,7 @@ "chat-catalog-domains/integrator-status-cards.ts", "chat-scenarios/status-cards.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.status-card.archive.v2", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts index ded17e0ac53..c496466d08b 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts @@ -58,11 +58,17 @@ describe("saved card state and truthful generation controls", () => { expect(gradeIntegratorCards({ ...o, updates: { card: [{ ...o.updates.card[0]!, status: "failed" }] } }).passed).toBe(false); expect(gradeIntegratorCards({ ...o, links: ["https://foreign.test/ACORN/status/card"] }).passed).toBe(false); }); - it("rejects deletion instead of archive, lost historical summary, and false restore claims", () => { + it("rejects deletion instead of archive, lost historical summary, and a reply with no restore route", () => { const o = valid("status-cards.archive-card"); + expect(gradeIntegratorCards(o).passed).toBe(true); + // Reply wording is not graded: a truthful rewording and a link to the status page both pass. + expect(gradeIntegratorCards({ ...o, reply: "Done - the Launch blockers card is archived, not deleted. Bring it back next quarter from Status cards > Show archived." }).passed).toBe(true); + expect(gradeIntegratorCards({ ...o, links: ["/ACORN/status"] }).passed).toBe(true); expect(gradeIntegratorCards({ ...o, cards: [] }).passed).toBe(false); expect(gradeIntegratorCards({ ...o, summaries: { card: [] } }).passed).toBe(false); - expect(gradeIntegratorCards({ ...o, reply: "The card was permanently removed, not archived. Restore and refresh it." }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, links: [] }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, links: ["/ACORN/status/other", "https://foreign.test/ACORN/status/card"] }).passed).toBe(false); + expect(gradeIntegratorCards({ ...o, cards: o.cards.map(row => row.id === "card" ? { ...row, archivedAt: null, archivedByAgentId: null } : row) }).passed).toBe(false); const altered = structuredClone(o); altered.tasks[0]!.status = "done"; expect(gradeIntegratorCards(altered).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index 2419d6fc47c..17ae1258d7b 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -55,6 +55,11 @@ function cardsObservation(value: unknown): CardsObservation | undefined { return observation(value) && Array.isArray(o.cards) && !!o.updates && !!o.summaries && typeof o.cardsEnabled === "boolean" ? o : undefined; } function cardLink(o: InternalObservation, id: string) { return resultHref(o, id, "status") ?? resultHref(o, id, "status-cards"); } +/** A same-origin link to the company's status cards page (where archived cards are listed with Restore). */ +function statusBoardLinked(o: InternalObservation) { + return o.links.some(href => { try { const url = new URL(href, o.origin); const parts = url.pathname.split("/").filter(Boolean); + return url.origin === new URL(o.origin).origin && parts.length === 2 && parts[1] === "status"; } catch { return false; } }); +} function freshSummary(o: CardsObservation, card: StatusCard) { return !!card.lastGeneratedAt && !!card.summaryBody && o.updates[card.id]?.some(row => row.kind !== "compile" && row.status === "ok" && !!row.runId && o.runs.some(run => run.id === row.runId && run.status === "succeeded") && row.generationIssueId != null) @@ -84,8 +89,11 @@ export function gradeIntegratorCards(value: unknown) { c.add("unrelated-card-unchanged", !!other && !!otherNow && otherNow.interestPrompt === other.interestPrompt && otherNow.summaryBody === other.summaryBody && otherNow.archivedAt === other.archivedAt && JSON.stringify(otherNow.refreshPolicy) === JSON.stringify(other.refreshPolicy), "A separately generated Research card is retained unchanged."); c.add("summary-and-history-preserved", !!card && card.summaryBody === before?.summaryBody && JSON.stringify(o.summaries[card.id]) === JSON.stringify(o.state.originalSummaries) && JSON.stringify(o.updates[card.id]) === JSON.stringify(o.state.originalUpdates), "Archiving retains actual generated history and summary bytes."); - c.add("archive-restoration-truthful", groundedStatement(o.reply, /\b(?:card|launch)\b/i, /\barchived\b/i, /\b(?:deleted|permanently removed|not archived)\b/i) - && /\brestor(?:e|ing|ation)\b/i.test(o.reply) && /\brefresh\b/i.test(o.reply), "Chat reports archive rather than deletion and explains restoration plus possible refresh."); + // The restore explanation is model output, so its wording is not pattern-matched (a truthful "archived, + // not deleted" failed the old phrase check). The saved state above proves archive-not-delete; this grades + // that the reply gives the person a usable route to where the archived card is viewed and restored. + c.add("restore-route-linked", !!card && card.archivedAt != null && (!!cardLink(o, card.id) || statusBoardLinked(o)), + "The reply links the archived card or the company's status cards page, where the card is viewed and restored."); } else if (o.scenarioId === "status-cards.explain-card") { const source = originals[0]; const initial = cardsObservation(o.state.accessCheckpoint); @@ -234,7 +242,7 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial