diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts index 810bb958163..41909882a98 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts @@ -1,13 +1,14 @@ -import { describe, expect, it } from "vitest"; -import { buildHistoricalWebsiteImport, gradeArtifactSearch, historicalWebsitePreviewMatches } from "./internal-artifact-search.js"; +import { describe, expect, it, vi } from "vitest"; +import { buildHistoricalWebsiteImport, gradeArtifactSearch, historicalWebsitePreviewMatches, readReturnedWebsiteFiles } from "./internal-artifact-search.js"; import { parse } from "yaml"; import { companyPortabilityService } from "../../../server/src/services/company-portability.js"; function valid() { const target = { id: "final-task", companyId: "company", title: "Final launch website delivery", status: "done", completedAt: "2026-10-21T12:00:00.000Z" }; const hidden = { id: "hidden-task", companyId: "company", title: "Restricted final website NIGHTJAR", status: "done" }; - const file = { id: "file", issueId: "final-task", assetId: "asset", issueCommentId: "delivery", sha256: "a".repeat(64), contentPath: "/api/attachments/file/content", createdAt: "2026-10-26T12:00:00.000Z" }; - return { scenarioId: "documents-artifacts.find-artifact", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(target), privateTask: hidden, privateMarker: hidden.title, privateAttachment: { id: "hidden-file", assetId: "hidden-asset" }, finalAttachment: structuredClone(file), finalHash: file.sha256, deliveryAt: target.completedAt, weekStart: Date.parse("2026-10-19T00:00:00Z"), weekEnd: Date.parse("2026-10-26T00:00:00Z"), originalTasks: structuredClone([target, hidden]), distractors: [hidden] }, - tasks: [target, hidden], projects: [], goals: [], comments: [], activity: [{ id: "import", action: "company.imported", createdAt: "2026-10-26T12:00:01.000Z" }], documents: [], revisions: [], runs: [], humanQueries: [], + const file = { companyId: "company", byteSize: 115, originalFilename: "final-website.html", id: "file", issueId: "final-task", assetId: "asset", issueCommentId: "delivery", sha256: "a".repeat(64), contentPath: "/api/attachments/file/content", createdAt: "2026-10-26T12:00:00.000Z" }; + const history = [{ issueId: target.id, comments: [{ id: "delivery", body: "original", createdAt: target.completedAt }], attachments: [structuredClone(file)] }, { issueId: hidden.id, comments: [], attachments: [] }]; + return { chatIssueId: "chat", returnedFiles: [{ attachment: structuredClone(file), downloadedHash: file.sha256, byteSize: 115 }], sourceHistory: structuredClone(history), scenarioId: "documents-artifacts.find-artifact", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(target), privateTask: hidden, privateMarker: hidden.title, privateAttachment: { id: "hidden-file", assetId: "hidden-asset" }, finalAttachment: structuredClone(file), finalHash: file.sha256, deliveryAt: target.completedAt, weekStart: Date.parse("2026-10-19T00:00:00Z"), weekEnd: Date.parse("2026-10-26T00:00:00Z"), originalHistory: structuredClone(history), originalTasks: structuredClone([target, hidden]), distractors: [hidden] }, + tasks: [target, hidden], projects: [], goals: [], comments: [], activity: [{ id: "import", action: "company.imported", createdAt: "2026-10-26T12:00:01.000Z" }], documents: [], revisions: [], runs: [{ id: "source-run", status: "succeeded" }], humanQueries: [], reply: "This is the final approved website delivered last week; the import uploaded the preserved file again today.", links: ["/RUN/issues/final-task", "/api/attachments/file/content?download=1"], finalAttachment: file, currentSource: target, downloadedHash: file.sha256, deliveryComments: [{ id: "delivery", createdAt: target.completedAt, body: `Final approved website delivered, SHA256 ${file.sha256}.` }], searchStatus: 200, searchResults: [{ id: "final-task" }], privateFileStatus: 403 }; } @@ -40,6 +41,52 @@ describe("migrated historical website artifact calibration", () => { const source = valid(); source.reply = bytes.reply; source.finalAttachment.issueId = "draft-task"; expect(gradeArtifactSearch(source).passed).toBe(false); }); + it("accepts an independently downloaded, attributed response copy without changing source history", () => { + const o = valid(); const copy = { ...o.finalAttachment, id: "copy", issueId: "chat", contentPath: "/api/attachments/copy/content", createdByAgentId: "ceo", originatingRunId: "source-run" }; + o.links[1] = copy.contentPath; o.returnedFiles = [{ attachment: copy, downloadedHash: copy.sha256, byteSize: copy.byteSize }]; + o.activity.push({ id: "copy-added", action: "issue.attachment_added", agentId: "ceo", entityId: "chat" } as never); + expect(gradeArtifactSearch(o).passed).toBe(true); + for (const change of [ + { downloadedHash: "b".repeat(64) }, { byteSize: 114 }, + { attachment: { ...copy, companyId: "foreign" } }, { attachment: { ...copy, issueId: "other-chat" } }, + { attachment: { ...copy, createdByAgentId: "other" } }, { attachment: { ...copy, originatingRunId: "old-run" } }, + { attachment: { ...copy, sha256: "b".repeat(64) } }, { attachment: { ...copy, originalFilename: "draft.html" } }, + ]) { const wrong = structuredClone(o); wrong.returnedFiles[0] = { ...wrong.returnedFiles[0]!, ...change }; expect(gradeArtifactSearch(wrong).passed).toBe(false); } + const absent = structuredClone(o); absent.returnedFiles = []; expect(gradeArtifactSearch(absent).passed).toBe(false); + }); + it("rejects historical task, comment, attachment, or completion mutation while allowing current-chat delivery", () => { + for (const mutate of [ + (o: ReturnType) => { o.tasks[0]!.title = "Changed"; }, + (o: ReturnType) => { (o.tasks[0]! as typeof o.currentSource).completedAt = "2026-10-22T12:00:00Z"; }, + (o: ReturnType) => { o.sourceHistory[0]!.comments[0]!.body = "changed"; }, + (o: ReturnType) => { o.sourceHistory[0]!.attachments[0]!.sha256 = "b".repeat(64); }, + (o: ReturnType) => { o.sourceHistory = []; }, + (o: ReturnType) => { o.activity.push({ id: "mutation", action: "issue.updated", agentId: "ceo", entityId: "final-task" } as never); }, + ]) { const o = valid(); mutate(o); expect(gradeArtifactSearch(o).passed).toBe(false); } + }); + it("does not mistake explicitly excluded other-source dates for the selected delivery date", () => { + const o = valid(); const old = { id: "old-task", identifier: "RUN-2", companyId: "company", title: "Earlier final", status: "done" }; + o.state.distractors.push(old); o.reply = "Found the final website delivered 2026-10-21.\nNot the others: RUN-2 old-final.html is from 2026-09-30, and the draft was never delivered."; + expect(gradeArtifactSearch(o).passed).toBe(true); + for (const reply of [ + "The final was delivered 2026-10-22.\nNot the others: RUN-2 from 2026-09-30.", + "The final wasn't delivered 2026-10-21.\nNot the others: RUN-2 from 2026-09-30.", + "The final was delivered 2026-10-21.\nNot the others: final-task was delivered 2026-10-22; RUN-2 from 2026-09-30.", + "The final was delivered 2026-10-21.\nNot the others: unknown-task from 2026-09-30.", + "The final was delivered 2026-10-21.\nNot the others: RUN-2 from 2026-09-30; the final was delivered 2026-10-22.", + ]) { const wrong = structuredClone(o); wrong.reply = reply; expect(gradeArtifactSearch(wrong).passed).toBe(false); } + }); + it("observes the actual rendered copy with authenticated byte reads and no raw-request fallback", async () => { + const o = valid(); const body = Buffer.from("actual copied file"); const hash = (await import("node:crypto")).createHash("sha256").update(body).digest("hex"); + o.state.finalHash = hash; o.state.finalAttachment.byteSize = body.length; + const copy = { ...o.finalAttachment, companyId: "company", id: "copy", issueId: "chat", contentPath: "/api/attachments/copy/content", sha256: hash, byteSize: body.length }; + o.links = [copy.contentPath]; const read = vi.fn().mockResolvedValue(body); const raw = vi.fn(() => { throw new Error("raw transport forbidden"); }); + const ctx = { issue: () => ({ id: "chat" }), input: { api: { get: vi.fn().mockImplementation((path: string) => Promise.resolve(path.includes("/chat/") ? [copy] : [o.finalAttachment])), getAttachmentContent: read, request: { get: raw } } } }; + const result = await readReturnedWebsiteFiles(ctx as never, o as never); + expect(read).toHaveBeenCalledExactlyOnceWith("copy"); expect(result).toEqual([{ attachment: copy, downloadedHash: hash, byteSize: body.length }]); expect(raw).not.toHaveBeenCalled(); + o.links = ["https://foreign.test" + copy.contentPath]; expect(await readReturnedWebsiteFiles(ctx as never, o as never)).toEqual([]); + o.links = [copy.contentPath]; read.mockRejectedValue(new Error("403")); await expect(readReturnedWebsiteFiles(ctx as never, o as never)).rejects.toThrow("403"); + }); it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); }); it("passes the actual production portability parser with all original dates/comments/attachments and no database access", async () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts index 1dbd860c88c..21921d54e01 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts @@ -4,14 +4,16 @@ import { expect } from "@playwright/test"; import { stringify } from "yaml"; import type { IssueAttachment } from "../../../packages/shared/src/types/issue.js"; import type { CompanyPortabilityFileEntry } from "../../../packages/shared/src/types/company-portability.js"; -import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { makeDefinition } from "./internal-definition.js"; import { mainPerson } from "./internal-privacy.js"; import { readSearchProbeResponse } from "./internal-search-response.js"; import { checks, entity, hasLink, observation, observeInternal, project, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; const sha = (body: string | Buffer) => createHash("sha256").update(body).digest("hex"); interface HistoricalTask extends Entity { completedAt?: string | null } -interface SearchObservation extends InternalObservation { finalAttachment: IssueAttachment; downloadedHash: string; deliveryComments: Entity[]; searchStatus: number; searchResults: unknown[]; privateFileStatus: number; currentSource: HistoricalTask } +interface ReturnedFile { attachment: IssueAttachment; downloadedHash: string; byteSize: number } +interface SourceHistory { issueId: string; comments: Entity[]; attachments: IssueAttachment[] } +interface SearchObservation extends InternalObservation { chatIssueId: string; returnedFiles: ReturnedFile[]; sourceHistory: SourceHistory[]; finalAttachment: IssueAttachment; downloadedHash: string; deliveryComments: Entity[]; searchStatus: number; searchResults: unknown[]; privateFileStatus: number; currentSource: HistoricalTask } export function buildHistoricalWebsiteImport(deliveryAt: string, nonce: string) { const specs = [ { slug: "final-website", title: "Final launch website delivery", status: "done", at: deliveryAt, filename: "final-website.html", label: "Final approved website delivered" }, @@ -53,8 +55,44 @@ export function historicalWebsitePreviewMatches(value: unknown, bundle: ReturnTy }); } function fileLink(o: InternalObservation, file: IssueAttachment) { - return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === file.contentPath; } catch { return false; } }); + return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && !url.username && !url.password && !url.hash && url.pathname === file.contentPath; } catch { return false; } }); } + +/** Read only actually rendered, same-origin attachment links through the current human's API actor. */ +export async function readReturnedWebsiteFiles(ctx: ChatCatalogContext, o: InternalObservation): Promise { + const chat = ctx.issue(); if (!chat) throw new Error("Artifact response has no actual chat"); + const source = entity(o.state, "task"); + const attachments = [...await ctx.input.api.get(`/api/issues/${source.id}/attachments`), + ...await ctx.input.api.get(`/api/issues/${chat.id}/attachments`)]; + const returned: ReturnedFile[] = []; + for (const attachment of attachments) { + if (!fileLink(o, attachment) || attachment.companyId !== o.companyId || attachment.sha256 !== o.state.finalHash + || attachment.byteSize !== (o.state.finalAttachment as IssueAttachment).byteSize) continue; + const bytes = await ctx.input.api.getAttachmentContent(attachment.id); + returned.push({ attachment, downloadedHash: sha(bytes), byteSize: bytes.byteLength }); + } + return returned; +} +function historySignature(history: SourceHistory) { + return JSON.stringify({ issueId: history.issueId, + comments: history.comments.map(row => [row.id, row.issueId, row.body, row.createdAt, row.authorAgentId, row.authorUserId]).sort(), + attachments: history.attachments.map(row => [row.id, row.issueId, row.issueCommentId, row.sha256, row.byteSize, row.originalFilename, row.createdAt]).sort() }); +} +function selectedDeliveryProse(o: SearchObservation, target: Entity, distractors: Entity[]) { + const mentions = (line: string, row: Entity) => [row.id, row.identifier, row.title].filter(Boolean).some(name => line.includes(name!)); + // A separately labelled exclusion names a known other source, not the selected final. + return o.reply.split(/\n+/).filter(line => !( /^\s*(?:[-*]\s*)?not the others?\s*:/i.test(line) + && distractors.some(row => mentions(line, row)) && !mentions(line, target) + && !/\b(?:the|this|selected|actual)\s+(?:approved\s+)?final\b/i.test(line))).join("\n"); +} +function verifiedReturnedFile(o: SearchObservation, target: Entity, original: IssueAttachment) { + return o.returnedFiles?.find(({ attachment: a, downloadedHash, byteSize }) => !!fileLink(o, a) + && a.companyId === o.companyId && a.sha256 === o.state.finalHash && downloadedHash === o.state.finalHash + && a.byteSize === original.byteSize && byteSize === original.byteSize && a.originalFilename === original.originalFilename + && (a.id === original.id && a.issueId === target.id || a.issueId === o.chatIssueId && a.createdByAgentId === o.leadId + && o.runs.some(run => run.id === a.originatingRunId && run.status === "succeeded"))); +} + /** Compare stated delivery dates with the preserved source date, not today's import date. */ export function describesHistoricalDelivery(reply: string, deliveredAt: string) { const expected = new Date(deliveredAt); @@ -99,8 +137,8 @@ export function gradeArtifactSearch(value: unknown) { && Date.parse(String(o.finalAttachment.createdAt)) > Date.parse(String(o.state.deliveryAt)), "The final belongs to the requested prior week through preserved completion and attachment-bound source-comment dates; the actual attachment import happened later."); c.add("exact-accessible-final-and-download", o.finalAttachment.id === file.id && o.finalAttachment.issueId === target.id && o.finalAttachment.sha256 === o.state.finalHash - && o.downloadedHash === o.state.finalHash && hasLink(o, "issues", target) && !!fileLink(o, o.finalAttachment) - && /\b(?:final|approved|delivered)\b/i.test(o.reply) && describesHistoricalDelivery(o.reply, String(o.state.deliveryAt)), + && o.downloadedHash === o.state.finalHash && hasLink(o, "issues", target) && !!verifiedReturnedFile(o, target, file) + && /\b(?:final|approved|delivered)\b/i.test(o.reply) && describesHistoricalDelivery(selectedDeliveryProse(o, target, (o.state.distractors as Entity[]) ?? []), String(o.state.deliveryAt)), "The answer selects the actual final from the requested source time, with both exact source and usable verified download links."); const distractors = o.state.distractors as Entity[] | undefined; c.add("draft-older-final-and-private-snippets-excluded", !!distractors && distractors.every(row => !hasLink(o, "issues", row) || row.id !== hidden.id && o.reply.split(/\n/).some(part => [row.id, row.identifier, row.title].filter(Boolean).some(name => part.includes(name!)) @@ -109,9 +147,16 @@ export function gradeArtifactSearch(value: unknown) { && !JSON.stringify(o.searchResults).includes(String(o.state.privateMarker)) && !o.reply.includes(String(o.state.privateMarker)) && !o.reply.includes(hidden.title ?? String(o.state.privateMarker)) && !o.links.some(href => href.includes((o.state.privateAttachment as IssueAttachment).id) || href.includes((o.state.privateAttachment as IssueAttachment).assetId)), "A newer unapproved draft and an older final are either omitted or clearly qualified rather than substituted, and independently denied private artifacts and previews remain outside results."); - const originals = o.state.originalTasks as Entity[] | undefined; + const originals = o.state.originalTasks as HistoricalTask[] | undefined; c.add("history-not-rewritten-by-search", !!originals && originals.every(row => sameTask(row, o.tasks.find(now => now.id === row.id))) - && !o.activity.some(row => row.agentId === o.leadId && (/issue\.(?:attachment_|work_product_)/.test(row.action ?? "") || originals.some(source => source.id === row.entityId) && /issue\.(?:updated|document_|deleted)/.test(row.action ?? ""))), + && originals.every(row => (o.tasks.find(now => now.id === row.id) as HistoricalTask | undefined)?.completedAt === row.completedAt) + && Array.isArray(o.state.originalHistory) && o.state.originalHistory.length === originals.length + && originals.every(original => (o.state.originalHistory as SourceHistory[]).filter(row => row.issueId === original.id).length === 1) + && (o.state.originalHistory as SourceHistory[]).every(before => { + const after = o.sourceHistory?.find(row => row.issueId === before.issueId); return !!after && historySignature(before) === historySignature(after); + }) + && !o.activity.some(row => row.agentId === o.leadId && originals.some(source => source.id === row.entityId) + && /issue\.(?:attachment_|work_product_|updated|document_|deleted|comment_)/.test(row.action ?? "")), "Searching preserves the migrated tasks, delivery state and source history."); return c.result(); } @@ -142,7 +187,8 @@ function findArtifact(): ChatCatalogCase { } if (!ctx.input.privateSecrets) throw new Error("Historical search requires private probe credential redaction"); key = await ctx.input.api.postSensitive(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `historical-search-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key!.token); - const state = { project: p, task: records[0], finalAttachment: files[0], finalHash: bundle.receipts[0]!.sha256, deliveryAt, weekStart, weekEnd, timezone, + const originalHistory = await Promise.all(records.map(async row => ({ issueId: row.id, comments: await ctx.input.api.get(`/api/issues/${row.id}/comments?order=asc`), attachments: await ctx.input.api.get(`/api/issues/${row.id}/attachments`) }))); + const state = { originalHistory, project: p, task: records[0], finalAttachment: files[0], finalHash: bundle.receipts[0]!.sha256, deliveryAt, weekStart, weekEnd, timezone, originalTasks: records, distractors: records.slice(1), privateTask: records[3], privateAttachment: files[3], privateMarker: `${ctx.input.nonce}-restricted-final` }; await ctx.input.evidence("chat-catalog-public-historical-delivery-import.json", { preview, imported, state, provenance: "Synthetic historical migration through supported public company import. Original source completion/comment dates are preserved; attachments are re-uploaded at import time, not falsely backdated." }); return state; }, gradeArtifactSearch); @@ -158,15 +204,17 @@ function findArtifact(): ChatCatalogCase { const hiddenUrl = new URL((state.privateAttachment as IssueAttachment).contentPath, ctx.input.api.baseURL); const denied = await fetch(hiddenUrl, { redirect: "error", signal: AbortSignal.timeout(15000), headers: { Authorization: `Bearer ${key.token}` } }); await denied.body?.cancel(); const results = search.body as { results?: unknown[] } | undefined; if (search.status === 200 && !Array.isArray(results?.results)) throw new Error("Search omitted its production result collection"); - return { ...o, currentSource, finalAttachment: attachments.find(row => row.id === file.id)!, downloadedHash: sha(bytes), deliveryComments, searchStatus: search.status, searchResults: results?.results ?? [], privateFileStatus: denied.status }; + const sourceHistory = await Promise.all((state.originalTasks as Entity[]).map(async row => ({ issueId: row.id, comments: await ctx.input.api.get(`/api/issues/${row.id}/comments?order=asc`), attachments: await ctx.input.api.get(`/api/issues/${row.id}/attachments`) }))); + const returnedFiles = await readReturnedWebsiteFiles(ctx, o); + return { ...o, chatIssueId: ctx.issue()!.id, returnedFiles, sourceHistory, currentSource, finalAttachment: attachments.find(row => row.id === file.id)!, downloadedHash: sha(bytes), deliveryComments, searchStatus: search.status, searchResults: results?.results ?? [], privateFileStatus: denied.status }; }, async verifyResult(ctx, state, value) { - const o = value as SearchObservation; await base.verifyResult(ctx, state, value); const href = fileLink(o, o.finalAttachment); if (!href) throw new Error("Missing exact historical file link"); + const o = value as SearchObservation; await base.verifyResult(ctx, state, value); const returned = verifiedReturnedFile(o, entity(state, "task"), state.finalAttachment as IssueAttachment); const href = returned && fileLink(o, returned.attachment); if (!href) throw new Error("Missing exact historical file link"); const page = await ctx.input.page.context().newPage(); try { const waiting = page.waitForEvent("download", { timeout: 5000 }).catch(() => undefined); await page.goto(new URL(href, o.origin).href).catch(error => { if (!/download|ERR_ABORTED/i.test(String(error))) throw error; }); const download = await waiting; if (download) { const path = await download.path(); if (!path || sha(await readFile(path)) !== state.finalHash) throw new Error("Historical browser download differs from preserved source bytes"); } else await expect(page.getByRole("heading", { name: "Final launch website delivery", exact: true })).toBeVisible(); - await ctx.input.evidence("chat-catalog-historical-file-opened.json", { attachmentId: o.finalAttachment.id, deliveryAt: state.deliveryAt, importedAt: o.finalAttachment.createdAt, sha256: o.downloadedHash }); + await ctx.input.evidence("chat-catalog-historical-file-opened.json", { attachmentId: returned!.attachment.id, sourceAttachmentId: o.finalAttachment.id, deliveryAt: state.deliveryAt, importedAt: o.finalAttachment.createdAt, sha256: returned!.downloadedHash }); } finally { await page.close(); } }, async cleanup(ctx) { if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } },