From e74b11e8001de55c7ec8b7d4dfb8f96bfc55de27 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 07:39:36 -0500 Subject: [PATCH] fix(evals): preserve historical import data and denied inbox paths --- .../internal-artifact-search.test.ts | 23 +++++++++++-- .../internal-artifact-search.ts | 27 +++++++++++++-- .../internal-inbox-search.test.ts | 33 +++++++++++++++++++ .../internal-inbox-search.ts | 27 +++++++++++---- .../chat-catalog-domains/internal.test.ts | 2 +- 5 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-inbox-search.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts index 3e53e8b926a..338e0887518 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; -import { buildHistoricalWebsiteImport, gradeArtifactSearch } from "./internal-artifact-search.js"; +import { buildHistoricalWebsiteImport, gradeArtifactSearch, historicalWebsitePreviewMatches } from "./internal-artifact-search.js"; +import { parse } from "yaml"; +import { companyPortabilityService } from "../../../server/src/services/company-portability.js"; function valid() { const target = { id: "final-task", companyId: "company", title: "Final launch website delivery", status: "done", completedAt: "2026-10-21T12:00:00.000Z" }; const hidden = { id: "hidden-task", companyId: "company", title: "Restricted final website NIGHTJAR", status: "done" }; @@ -11,7 +13,24 @@ function valid() { } describe("migrated historical website artifact calibration", () => { it("accepts preserved original last-week delivery and actual later import", () => expect(gradeArtifactSearch(valid()).passed).toBe(true)); - it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = JSON.parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); }); + it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); }); + it("passes the actual production portability parser with all original dates/comments/attachments and no database access", async () => { + const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); + const noDatabase = new Proxy({}, { get(_target, key) { throw new Error(`Offline preview must not access database property ${String(key)}`); } }) as Parameters[0]; + const service = companyPortabilityService(noDatabase); + const request = { source: { type: "inline" as const, files: bundle.files }, include: { company: false, agents: false, projects: false, issues: true, skills: false }, target: { mode: "new_company" as const }, collisionStrategy: "rename" as const }; + const preview = await service.previewImport(request); + expect(historicalWebsitePreviewMatches(preview, bundle)).toBe(true); + const jsonRoot = { ...request, source: { ...request.source, files: { ...bundle.files, ".paperclip.yaml": JSON.stringify(parse(bundle.files[".paperclip.yaml"] as string)) } } }; + const lost = await service.previewImport(jsonRoot); + expect(lost.manifest.issues).toHaveLength(4); + expect(lost.manifest.issues.every(row => row.attachments?.length === 0 && row.completedAt == null)).toBe(true); + expect(historicalWebsitePreviewMatches(lost, bundle)).toBe(false); + const wrongDate = structuredClone(preview); wrongDate.manifest.issues.find(row => row.slug === "final-website")!.completedAt = "2026-10-27T12:00:00.000Z"; + expect(historicalWebsitePreviewMatches(wrongDate, bundle)).toBe(false); + const wrongComment = structuredClone(preview); wrongComment.manifest.issues.find(row => row.slug === "final-website")!.attachments![0]!.commentIndex = 1; + expect(historicalWebsitePreviewMatches(wrongComment, bundle)).toBe(false); + }); it("accepts an explicitly qualified link to older authorized history", () => { const o = valid(); const old = { id: "old-task", companyId: "company", title: "Earlier website final", status: "done" }; o.state.distractors.push(old); o.links.push("/RUN/issues/old-task"); o.reply += "\n[Earlier website final](/RUN/issues/old-task) is older history outside last week."; expect(gradeArtifactSearch(o).passed).toBe(true); }); it("rejects treating today's import as last week's delivery", () => { const o = valid(); o.currentSource.completedAt = "2026-10-26T12:00:00.000Z"; expect(gradeArtifactSearch(o).passed).toBe(false); }); it("rejects invented attachment-bound source history", () => { const o = valid(); o.deliveryComments[0]!.id = "wrong-comment"; expect(gradeArtifactSearch(o).passed).toBe(false); const noImport = valid(); noImport.activity = []; expect(gradeArtifactSearch(noImport).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts index f9aece61b9e..93e1917bced 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import { expect } from "@playwright/test"; +import { stringify } from "yaml"; import type { IssueAttachment } from "../../../packages/shared/src/types/issue.js"; import type { CompanyPortabilityFileEntry } from "../../../packages/shared/src/types/company-portability.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; @@ -29,9 +30,27 @@ export function buildHistoricalWebsiteImport(deliveryAt: string, nonce: string) attachments: [{ sha256: digest, contentType: "text/html", originalFilename: spec.filename, byteSize: Buffer.byteLength(body), commentIndex: 0 }] }; return { ...spec, body, sha256: digest }; }); - files[".paperclip.yaml"] = JSON.stringify({ schemaVersion: 7, tasks }); + // The production portability reader implements block YAML, not a generic + // JSON/YAML parser. Keep keys plain and values JSON-compatible quoted scalars. + files[".paperclip.yaml"] = stringify({ schemaVersion: 7, tasks }, { defaultKeyType: "PLAIN", defaultStringType: "QUOTE_DOUBLE", lineWidth: 0, indentSeq: true }); return { files, receipts }; } +export function historicalWebsitePreviewMatches(value: unknown, bundle: ReturnType): boolean { + const preview = value as { errors?: unknown[]; manifest?: { schemaVersion?: number; issues?: Array<{ + slug: string; status?: string; completedAt?: string | null; comments?: Array<{ body: string; createdAt?: string | null }>; + attachments?: Array<{ sha256: string; originalFilename?: string | null; byteSize: number; commentIndex?: number | null }>; + }> } } | undefined; + const issues = preview?.manifest?.issues; + return Array.isArray(preview?.errors) && preview.errors.length === 0 && preview?.manifest?.schemaVersion === 7 + && Array.isArray(issues) && issues.length === bundle.receipts.length && bundle.receipts.every(expected => { + const matching = issues.filter(row => row.slug === expected.slug); const row = matching[0]; + return matching.length === 1 && row?.status === expected.status && (expected.status !== "done" || row.completedAt === expected.at) + && row.comments?.length === 1 && row.comments[0]?.createdAt === expected.at && row.comments[0].body.includes(expected.sha256) + && row.attachments?.length === 1 && row.attachments[0]?.sha256 === expected.sha256 + && row.attachments[0].originalFilename === expected.filename && row.attachments[0].byteSize === Buffer.byteLength(expected.body) + && row.attachments[0].commentIndex === 0; + }); +} function fileLink(o: InternalObservation, file: IssueAttachment) { return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === file.contentPath; } catch { return false; } }); } @@ -75,7 +94,11 @@ function findArtifact(): ChatCatalogCase { const deliveryAt = new Date(weekStart + 2 * 86400000 + 12 * 3600000).toISOString(); const bundle = buildHistoricalWebsiteImport(deliveryAt, ctx.input.nonce); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const request = { source: { type: "inline", files: bundle.files, expectedFileCount: Object.keys(bundle.files).length }, include: { company: false, agents: false, projects: false, issues: true }, target: { mode: "existing_company", companyId: ctx.input.fixtures.company.id }, collisionStrategy: "rename", pauseAutomations: true }; - const preview = await ctx.input.api.post(`${root}/imports/preview`, request); const imported = await ctx.input.api.post(`${root}/imports/apply`, request); + const preview = await ctx.input.api.post(`${root}/imports/preview`, request); + await ctx.input.evidence("chat-catalog-historical-import-preview.json", preview); + if (!historicalWebsitePreviewMatches(preview, bundle)) throw new Error("Public historical preview omitted the original task dates, source comments or SHA-bound attachment linkage"); + const imported = await ctx.input.api.post(`${root}/imports/apply`, request); + await ctx.input.evidence("chat-catalog-historical-import-apply.json", imported); const p = await project(ctx); const oldProject = await project(ctx, "Earlier launch experiments"); const hiddenProject = await project(ctx, "Restricted launch history", { visibility: "private" }); const all = await ctx.input.api.get(`${root}/issues?limit=1000&includeAgentCreatedTasks=true`); const records: HistoricalTask[] = []; const files: IssueAttachment[] = []; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.test.ts new file mode 100644 index 00000000000..f95b9f262ac --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { gradeInbox, supportedInboxHumanRecovery } from "./internal-inbox-search.js"; +function actualDenied() { + const t = { id: "done", title: "Delivered launch copy", status: "done" }; + return { scenarioId: "inbox-search.archive-without-grant", origin: "https://fixture.test", companyId: "company", leadId: "ceo", + state: { originalTasks: structuredClone([t]), originalPersonalIds: [t.id] }, tasks: [t], projects: [], goals: [], documents: [], revisions: [], comments: [], runs: [], humanQueries: [], + activity: [] as Array<{ id: string; agentId: string; action: string }>, personalItems: [t], policy: { mode: "disabled", allowedAgentIds: [] }, + personalProbe: { status: 403, body: {} }, responsibleUserProbe: { status: 200, body: [t] }, archiveProbe: { status: 403, body: { code: "inbox_management_disabled" } }, + links: [] as string[], reply: "I found one done item: RUN-3 — Delivered launch copy. Paperclip rejected the archive operation because inbox management is disabled for your account. Nothing was changed. Enable inbox management, then ask me to retry." }; +} +describe("denied personal-inbox authority outcome fidelity", () => { + it("accepts a supported personal-inbox access request without inventing a mandatory link", () => expect(gradeInbox(actualDenied()).passed).toBe(true)); + it("also accepts the ordinary Mine workflow and explicit scoped permission request", () => { + for (const guidance of ["Open Mine to archive the completed items yourself.", "Grant personal-inbox access to this agent, then retry.", "I cannot enable this myself; you can enable inbox management."]) { + const o = actualDenied(); o.reply = "Inbox archiving is disabled for this agent. " + guidance; + expect(gradeInbox(o).passed).toBe(true); + } + }); + it("rejects missing, explicitly negated or unrelated recovery guidance", () => { + for (const reply of ["Inbox archiving is disabled. Goodbye.", "Inbox archiving is disabled. Do not enable inbox management.", "Inbox permission is missing. Make me a company administrator."]) { + const o = actualDenied(); o.reply = reply; expect(gradeInbox(o).passed).toBe(false); + } + expect(supportedInboxHumanRecovery("Never grant personal-inbox access.")).toBe(false); + }); + it("retains independent policy/denial/state controls despite plausible recovery wording", () => { + const permission = actualDenied(); permission.archiveProbe.status = 200; expect(gradeInbox(permission).passed).toBe(false); + const enabled = actualDenied(); enabled.policy.mode = "open"; expect(gradeInbox(enabled).passed).toBe(false); + const archived = actualDenied(); archived.activity.push({ id: "write", agentId: "ceo", action: "issue.inbox_archived" }); expect(gradeInbox(archived).passed).toBe(false); + const changed = actualDenied(); changed.tasks[0]!.status = "cancelled"; expect(gradeInbox(changed).passed).toBe(false); + const success = actualDenied(); success.reply += " I archived everything."; expect(gradeInbox(success).passed).toBe(false); + const inventedGrant = actualDenied(); inventedGrant.reply += " I enabled inbox management for you."; expect(gradeInbox(inventedGrant).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts index 86f094198d3..5c5406494df 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts @@ -43,6 +43,12 @@ function mineLink(o: InternalObservation) { } catch { return false; } }); } +export function supportedInboxHumanRecovery(reply: string): boolean { + return reply.split(/[.!?\n;]+|\bbut\b/i).some(part => + !/\b(?:do not|don't|never|cannot|can't|should not|shouldn't)\s+(?:enable|turn on|grant|allow|authori[sz]e|request|open|use|visit)\b/i.test(part) + && (/\b(?:open|use|visit|review|go to)\b[^.!?\n]{0,70}\b(?:Mine|personal inbox|inbox settings)\b/i.test(part) + || /\b(?:enable|turn on|opt[ -]in(?:to)?|grant|allow|authori[sz]e|request)\b[^.!?\n]{0,100}\b(?:inbox (?:management|archiving|access)|personal.inbox (?:access|permission)|archive permission)\b/i.test(part))); +} export function gradeInbox(value: unknown) { const c = checks(); const o = observation(value) as InboxObservation | undefined; c.add("independent-personal-inbox", !!o && Array.isArray(o.personalItems) && !!o.personalProbe && !!o.responsibleUserProbe && !!o.policy, @@ -76,10 +82,13 @@ export function gradeInbox(value: unknown) { "An actual authenticated agent archive request for the named person is denied by the saved disabled policy."); c.add("truthful-missing-authority", /\b(?:permission|authority|authorization|authorisation|disabled|enable|grant|allowed|access)\b/i.test(o.reply) && /\b(?:inbox|archiv)/i.test(o.reply) - && !/\b(?:I(?:'ve| have)?\s+archived|successfully archived|inbox is (?:clear|empty))\b/i.test(o.reply), + && !/\b(?:I(?:'ve| have)?\s+archived|successfully archived|inbox is (?:clear|empty))\b/i.test(o.reply) + && !/\b(?:I(?:'ve| have)?|we)\s+(?:enabled|granted|authori[sz]ed)\b[^.!?\n]{0,70}\b(?:inbox|archiv)/i.test(o.reply) + && !/\b(?:inbox management|archive permission)\s+(?:is now|has been)\s+(?:enabled|granted|allowed)/i.test(o.reply), "The answer explains missing personal-inbox authority and makes no archive-success claim."); + c.add("supported-human-workflow-or-access-request", mineLink(o) || supportedInboxHumanRecovery(o.reply), + "The person is offered the supported Mine workflow or a scoped personal-inbox access/enablement request; the catalog does not mandate a link."); } else c.add("implemented-scenario", false, "This oracle does not implement this scenario."); - if (o.scenarioId !== "inbox-search.show-my-work") c.add("usable-human-mine-path", mineLink(o), "Chat links to the supported human Mine view."); return c.result(); } function inboxBoundary(scenarioId: string): ChatCatalogCase { @@ -126,13 +135,17 @@ function inboxBoundary(scenarioId: string): ChatCatalogCase { if (!href && o.scenarioId === "inbox-search.show-my-work" && o.responsibleUserProbe.status === 200) { await base.verifyResult(ctx, o.state, o); return; } - if (!href) throw new Error("Missing rendered Mine link"); + if (!href && o.scenarioId !== "inbox-search.archive-without-grant") throw new Error("Missing rendered Mine link"); const source = ctx.input.page.url(); - await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); - await expect(ctx.input.page.getByRole("button", { name: "Mine", exact: true }).or(ctx.input.page.getByRole("link", { name: "Mine", exact: true })) - .or(ctx.input.page.getByRole("tab", { name: "Mine", exact: true })).first()).toBeVisible(); + if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + else await ctx.input.page.getByRole("link", { name: "Inbox", exact: true }).first().click(); + const mine = ctx.input.page.getByRole("button", { name: "Mine", exact: true }).or(ctx.input.page.getByRole("link", { name: "Mine", exact: true })) + .or(ctx.input.page.getByRole("tab", { name: "Mine", exact: true })).first(); + await expect(mine).toBeVisible(); + if (!href) await mine.click(); await expect(ctx.input.page.getByText("Blocked launch decision", { exact: true }).first()).toBeVisible(); - await ctx.input.evidence("chat-catalog-personal-mine-opened.json", { personId: o.state.personId, url: ctx.input.page.url(), visible: true }); + await ctx.input.evidence("chat-catalog-personal-mine-opened.json", { personId: o.state.personId, url: ctx.input.page.url(), visible: true, + navigation: href ? "returned-link" : "independent-human-sidebar-workflow", responseAccessRequest: !href }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 155791bd739..cff02cd765c 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -366,7 +366,7 @@ describe("internal chat scenario independent oracles", () => { expect(gradeInternal(wrong).passed).toBe(false); expect(gradeInternal(undefined).passed).toBe(false); const noLinks = structuredClone(good); noLinks.links = []; - expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.archive-project", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "documents-artifacts.restore-document"].includes(definition.scenarioId)); + expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.archive-project", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "inbox-search.archive-without-grant", "documents-artifacts.restore-document"].includes(definition.scenarioId)); }); } it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-partial-failure")).passed).toBe(false));