fix(evals): select existing document sources in standing chat

tests/runner-e2e/chat-catalog-domains/internal-document-context.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { IssueAttachment } from "../../../packages/shared/src/types/issue.js";
+import { selectDocumentSource } from "./internal-document-context.js";
+
+function fixture() {
+  const target = { id: "target", companyId: "company", projectId: "project", identifier: "RUN-1", title: "Website task" };
+  const source = { id: "chat", companyId: "company", projectId: null as string | null, description: null as string | null };
+  const document = { id: "doc", issueId: "target", key: "plan", title: "Implementation plan", latestRevisionId: "revision", body: "Original pricing bytes" };
+  const annotation = { id: "comment", documentId: "doc", originalRevisionId: "revision", status: "open" };
+  const attachment = { id: "file", issueId: "target", sha256: "original-hash", originalFilename: "website-v1.html", contentPath: "/api/attachments/file/content" } as IssueAttachment;
+  const api = {
+    get: vi.fn(async (path: string) => {
+      if (path === "/api/issues/target") return structuredClone(target);
+      if (path === "/api/issues/chat") return structuredClone(source);
+      if (path.endsWith("/annotations?status=all&includeComments=true")) return [structuredClone(annotation)];
+      if (path.endsWith("/documents/plan")) return structuredClone(document);
+      if (path.endsWith("/attachments")) return [structuredClone(attachment)];
+      throw new Error(`Unexpected read ${path}`);
+    }),
+    post: vi.fn(async () => ({})),
+    patch: vi.fn(async (_path: string, value: object) => Object.assign(source, value)),
+  };
+  const evidence = vi.fn(async () => undefined);
+  const ctx = { input: { api, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } }, evidence },
+    refreshIssue: vi.fn(async () => undefined), issue: () => source } as unknown as ChatCatalogContext;
+  return { ctx, target, source, document, annotation, attachment, api, evidence };
+}
+
+describe("factual visible document source selection", () => {
+  it("persists the selected task/document/comment on the real chat without copying an answer or editing the target", async () => {
+    const f = fixture(); const original = structuredClone({ document: f.document, annotation: f.annotation });
+    const receipt = await selectDocumentSource(f.ctx, { task: f.target, document: f.document, annotationId: f.annotation.id });
+    expect(f.source.description).toBe("Selected source: [Website task](/RUN/issues/RUN-1).\n\nSelected document: [Implementation plan](/RUN/issues/RUN-1#document-plan).\n\nSelected review comment: comment on this document.");
+    expect(receipt.targetIssueId).toBe("target");
+    expect({ document: f.document, annotation: f.annotation }).toEqual(original);
+    expect(f.api.patch).toHaveBeenCalledExactlyOnceWith("/api/issues/chat", { projectId: "project", description: f.source.description });
+    expect(f.api.post).toHaveBeenCalledExactlyOnceWith("/api/companies/company/chats/ceo");
+    expect(f.evidence).toHaveBeenCalledOnce();
+  });
+  it("selects an existing revision without supplying which historical contents to restore", async () => {
+    const f = fixture(); await selectDocumentSource(f.ctx, { task: f.target, document: f.document });
+    expect(f.source.description).toContain("#document-plan");
+    expect(f.source.description).not.toMatch(/pricing|revision|restore|previous|Original/);
+  });
+  it("selects the actual original delivery without uploading or revising any bytes", async () => {
+    const f = fixture(); await selectDocumentSource(f.ctx, { task: f.target, attachment: f.attachment });
+    expect(f.source.description).toContain("[website-v1.html](/api/attachments/file/content)");
+    expect(f.source.description).not.toMatch(/headline|shorten|new version/);
+    expect(f.api.patch.mock.calls.every(([path]) => path === "/api/issues/chat")).toBe(true);
+  });
+  it("names the generated workspace source without pre-registering the requested delivery", async () => {
+    const f = fixture(); await selectDocumentSource(f.ctx, { task: f.target, workspaceFilename: "website.html" });
+    expect(f.source.description).toContain("Selected workspace file: website.html");
+    expect(f.source.description).not.toMatch(/attachments|download|upload|register/);
+    expect(f.api.post).toHaveBeenCalledOnce();
+  });
+  it.each(["company", "project", "revision", "annotation", "attachment", "persist"]) ("rejects plausible wrong %s evidence before the canonical turn", async kind => {
+    const f = fixture(); const input = { task: structuredClone(f.target), document: { ...f.document }, annotationId: "comment", attachment: { ...f.attachment } };
+    if (kind === "company") f.target.companyId = "foreign";
+    if (kind === "project") f.target.projectId = "other-project";
+    if (kind === "revision") f.document.latestRevisionId = "other-revision";
+    if (kind === "annotation") f.annotation.documentId = "other-doc";
+    if (kind === "attachment") f.attachment.sha256 = "different-bytes";
+    if (kind === "persist") f.api.patch.mockImplementation(async () => f.source);
+    await expect(selectDocumentSource(f.ctx, input)).rejects.toThrow();
+    expect(f.evidence).not.toHaveBeenCalled();
+  });
+});

tests/runner-e2e/chat-catalog-domains/internal-document-context.ts

@@ -0,0 +1,55 @@
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { IssueDocument, IssueAttachment } from "../../../packages/shared/src/types/issue.js";
+import type { Entity } from "./internal-support.js";
+
+/** Ordinary, visible source selection on the real standing conversation. Only
+ * existing identities are supplied: no requested edit, answer or tool recipe. */
+export async function selectDocumentSource(ctx: ChatCatalogContext, input: {
+  task: Entity;
+  document?: { id: string; key: string; latestRevisionId: string | null };
+  annotationId?: string;
+  attachment?: IssueAttachment;
+  workspaceFilename?: string;
+}) {
+  const { api, fixtures } = ctx.input;
+  const target = await api.get<Entity>(`/api/issues/${input.task.id}`);
+  if (target.id !== input.task.id || target.companyId !== fixtures.company.id || target.projectId !== input.task.projectId)
+    throw new Error("Selected document source does not belong to the expected task/company/project");
+  const taskHref = `/${fixtures.company.issuePrefix}/issues/${target.identifier ?? target.id}`;
+  const lines = [`Selected source: [${target.title}](${taskHref}).`];
+  if (input.document) {
+    const doc = await api.get<IssueDocument>(`/api/issues/${target.id}/documents/${encodeURIComponent(input.document.key)}`);
+    if (doc.id !== input.document.id || doc.issueId !== target.id || doc.latestRevisionId !== input.document.latestRevisionId)
+      throw new Error("Selected source document or revision changed before the canonical turn");
+    lines.push(`Selected document: [${doc.title ?? doc.key}](${taskHref}#document-${encodeURIComponent(doc.key)}).`);
+    if (input.annotationId) {
+      const annotations = await api.get<Entity[]>(`/api/issues/${target.id}/documents/${encodeURIComponent(doc.key)}/annotations?status=all&includeComments=true`);
+      const annotation = annotations.find(row => row.id === input.annotationId);
+      if (!annotation || annotation.documentId !== doc.id || annotation.originalRevisionId !== doc.latestRevisionId || annotation.status !== "open")
+        throw new Error("Selected review annotation is not open on the actual source revision");
+      lines.push(`Selected review comment: ${annotation.id} on this document.`);
+    }
+  } else if (input.annotationId) throw new Error("A selected annotation requires its actual document");
+  if (input.attachment) {
+    const file = (await api.get<IssueAttachment[]>(`/api/issues/${target.id}/attachments`)).find(row => row.id === input.attachment!.id);
+    if (!file || file.issueId !== target.id || file.sha256 !== input.attachment.sha256 || file.contentPath !== input.attachment.contentPath)
+      throw new Error("Selected prior delivery is not the original source attachment");
+    lines.push(`Selected existing delivery: [${file.originalFilename}](${file.contentPath}).`);
+  }
+  if (input.workspaceFilename) lines.push(`Selected workspace file: ${input.workspaceFilename}, in the selected task's authorized workspace.`);
+  const description = lines.join("\n\n");
+  await api.post(`/api/companies/${fixtures.company.id}/chats/${fixtures.agent.id}`);
+  await ctx.refreshIssue();
+  const source = ctx.issue();
+  if (!source || source.id === target.id) throw new Error("Source selection requires the actual separate standing conversation");
+  await api.patch(`/api/issues/${source.id}`, { projectId: target.projectId, description });
+  const saved = await api.get<Entity>(`/api/issues/${source.id}`);
+  if (saved.id !== source.id || saved.companyId !== fixtures.company.id || saved.projectId !== target.projectId || saved.description !== description)
+    throw new Error("Visible source selection did not persist on the canonical conversation");
+  await ctx.refreshIssue();
+  const receipt = { sourceIssueId: saved.id, targetIssueId: target.id, projectId: target.projectId, description,
+    documentId: input.document?.id, revisionId: input.document?.latestRevisionId, annotationId: input.annotationId,
+    attachmentId: input.attachment?.id, attachmentSha256: input.attachment?.sha256, workspaceFilename: input.workspaceFilename };
+  await ctx.input.evidence("chat-catalog-selected-document-source.json", receipt);
+  return receipt;
+}

tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts

@@ -4,6 +4,8 @@ import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { privateDownloadCases } from "./internal-private-download.js";
 import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity } from "./internal-support.js";
 import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js";
+import { selectDocumentSource } from "./internal-document-context.js";
+import type { IssueDocument } from "../../../packages/shared/src/types/issue.js";
 
 export function grade_documents_artifacts(value: unknown) {
   const c = checks(); const o = observation(value);
@@ -78,7 +80,9 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [
       const saved = await putDocument(ctx, t.id, body, revisions.at(-1)?.id ?? null);
       revisions.push({ id: saved.latestRevisionId!, body });
     }
-    return { project: p, task: t, originalRevisions: revisions };
+    const document = await ctx.input.api.get<IssueDocument>(`/api/issues/${t.id}/documents/plan`);
+    const selectedSource = await selectDocumentSource(ctx, { task: t, document });
+    return { project: p, task: t, originalRevisions: revisions, selectedSource };
   })),
 
 ];
@@ -103,7 +107,9 @@ function resolveFeedback(): ChatCatalogCase {
     };
     const pricingThread = await createAnnotation(pricing, "Clarify that $19 is the monthly total for the team, not per person. Keep the amount and billing cadence.");
     const siblingThread = await createAnnotation(supportText, "Please add a support contact later; leave this unrelated comment open for now.");
-    return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId };
+    const selectedSource = await selectDocumentSource(ctx, { task: t,
+      document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id });
+    return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource };
   });
   return { ...base, sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
     const o = await observeInternal(ctx, state);

tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts

@@ -9,6 +9,7 @@ import { pollUntil } from "../api.js";
 import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { fixtureWorker, makeDefinition } from "./internal-definition.js";
 import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
+import { selectDocumentSource } from "./internal-document-context.js";
 interface Downloaded { id: string; sha256: string; body: string }
 interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[] }
 const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
@@ -74,12 +75,15 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
       const [original] = await readRegisteredArtifacts(ctx.input.api, t.id, priorRunId, [filename]);
       if (original.content !== actual) throw new Error("The prior registered delivery does not match the generated website");
       originalAttachment = (await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${t.id}/attachments`)).find(file => file.id === original.attachmentId)!; originalProductId = original.workProductId;
+      if (!originalAttachment) throw new Error("The original registered delivery attachment is missing from its source task");
     } else if ((await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${t.id}/attachments`)).length) throw new Error("The undelivered prerequisite unexpectedly uploaded the requested outcome");
     // The delivered revision source can be handed over without starting work. The
     // undelivered prerequisite stays honestly blocked with its original worker;
     // assigning a nonterminal task here could admit an undeclared extra run.
     const handedOff = revising ? await ctx.input.api.patch<typeof t>(`/api/issues/${t.id}`, { assigneeAgentId: ctx.input.fixtures.agent.id }) : ready.task;
-    const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId };
+    const selectedSource = await selectDocumentSource(ctx, { task: handedOff,
+      ...(revising ? { attachment: originalAttachment } : { workspaceFilename: filename }) });
+    const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource };
     await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state;
   }, gradeWebsiteDelivery);
   return { ...base, expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,