test(evals): prove case work links and current principal access

tests/runner-e2e/chat-catalog-domains/integrator-case-related.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { gradeIntegratorRelatedWork } from "./integrator-case-related.js";
+function valid() {
+  const primary = { id: "primary", identifier: "ACN-1", title: "Investigation", status: "done" };
+  const followup = { id: "followup", identifier: "ACN-2", title: "Mobile follow-up", status: "backlog" };
+  const row = { id: "case", identifier: "ACN-C1", workIssueId: primary.id, status: "in_progress", handoffVersion: 1 };
+  return { scenarioId: "cases.related-work", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { case: structuredClone(row), followup,
+    unrelated: { id: "same-name", identifier: "ACN-3", title: followup.title }, privateTask: { id: "foreign", identifier: "PRV-1", title: "Private customer" }, privateMarker: "PRIVATE-SENTINEL" },
+    tasks: [primary, followup], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], runs: [],
+    links: ["/ACN/issues/ACN-1", "/ACN/issues/ACN-2"], caseDetail: { ...row, issueLinks: [{ issueId: "primary", role: "work" }, { issueId: "followup", role: "reference" }] },
+    reverseLinks: { primary: [{ role: "work", case: structuredClone(row) }], followup: [{ role: "reference", case: structuredClone(row) }] }, deniedStatus: 404,
+    reply: "Primary investigation ACN-1 is completed. The linked follow-up ACN-2 remains backlog; the case is still open." };
+}
+describe("case relationships rather than similar names", () => {
+  it("accepts real primary/reference/reverse links and substantive state paraphrase", () => {
+    expect(gradeIntegratorRelatedWork(valid()).passed).toBe(true);
+    expect(gradeIntegratorRelatedWork({ ...valid(), reply: "The main Investigation task is done. Its reference Mobile follow-up is in backlog." }).passed).toBe(true);
+    expect(gradeIntegratorRelatedWork(undefined).passed).toBe(false);
+    expect(gradeIntegratorRelatedWork({ ...valid(), reverseLinks: {} }).passed).toBe(false);
+  });
+  it("rejects inferred same-name relationships, private exposure and wrong principal access", () => {
+    const o = valid();
+    expect(gradeIntegratorRelatedWork({ ...o, links: [...o.links, "/ACN/issues/ACN-3"] }).passed).toBe(false);
+    expect(gradeIntegratorRelatedWork({ ...o, reply: `${o.reply} Private customer: PRIVATE-SENTINEL.` }).passed).toBe(false);
+    expect(gradeIntegratorRelatedWork({ ...o, deniedStatus: 200 }).passed).toBe(false);
+    const missing = valid(); missing.caseDetail.issueLinks = [];
+    expect(gradeIntegratorRelatedWork(missing).passed).toBe(false);
+  });
+  it("rejects premature completion or relationship mutation", () => {
+    const o = valid();
+    expect(gradeIntegratorRelatedWork({ ...o, reply: `${o.reply} All case work is complete.` }).passed).toBe(false);
+    expect(gradeIntegratorRelatedWork({ ...o, reply: "Primary ACN-1 is not completed and remains backlog. Linked follow-up ACN-2 is done." }).passed).toBe(false);
+    const changed = valid(); changed.caseDetail.handoffVersion = 2;
+    expect(gradeIntegratorRelatedWork(changed).passed).toBe(false);
+    expect(gradeIntegratorRelatedWork({ ...o, activity: [{ id: "edit", agentId: "ceo", action: "case.updated" }] }).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/integrator-case-related.ts

@@ -0,0 +1,112 @@
+import { expect } from "@playwright/test";
+import type { cases } from "../../../packages/db/src/schema/cases.js";
+import type { HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { pollUntil } from "../api.js";
+import { checks, hasLink, observation, project, task, type Entity, type InternalObservation } from "./internal-support.js";
+import { fixtureWorker } from "./internal-definition.js";
+import { integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js";
+type CaseRecord = typeof cases.$inferSelect;
+interface CaseDetail extends CaseRecord { issueLinks: Array<{ issueId: string; role: string; issue: Entity }> }
+interface RelatedObservation extends InternalObservation { caseDetail: CaseDetail; reverseLinks: Record<string, Array<{ role: string; case: Pick<CaseRecord, "id" | "identifier" | "workIssueId"> }>>; deniedStatus: number }
+const resources = new WeakMap<ChatCatalogContext, { experimental: boolean; foreignCompanyId?: string }>();
+async function cleanup(ctx: ChatCatalogContext) {
+  const saved = resources.get(ctx); if (!saved) return;
+  if (saved.foreignCompanyId) await ctx.input.api.delete(`/api/companies/${saved.foreignCompanyId}`, { allowNotFound: true });
+  await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: saved.experimental }); resources.delete(ctx);
+}
+async function observeRelated(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<RelatedObservation> {
+  const base = await observeInternal(ctx, state); const row = state.case as CaseRecord;
+  const caseDetail = await ctx.input.api.get<CaseDetail>(`/api/cases/${row.id}`);
+  const ids = [row.workIssueId!, (state.followup as Entity).id];
+  return { ...base, caseDetail, deniedStatus: Number(state.deniedStatus), reverseLinks: Object.fromEntries(await Promise.all(ids.map(async id => [id, await ctx.input.api.get<RelatedObservation["reverseLinks"][string]>(`/api/issues/${id}/cases`)]))) };
+}
+function relatedObservation(value: unknown): RelatedObservation | undefined {
+  const o = value as RelatedObservation;
+  return observation(value) && !!o.caseDetail && Array.isArray(o.caseDetail.issueLinks) && !!o.reverseLinks && typeof o.deniedStatus === "number" ? o : undefined;
+}
+function reportsTaskState(reply: string, task: Entity, state: "done" | "backlog") {
+  const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const names = [task.identifier, task.title].filter((name): name is string => !!name);
+  const subject = new RegExp(names.map(escape).join("|"), "i");
+  const yes = state === "done" ? /\b(?:done|completed|finished)\b/i : /\bbacklog\b/i;
+  const wrong = state === "done" ? /\b(?:not (?:done|completed|finished)|still (?:running|pending))\b/i : /\b(?:not (?:in )?backlog|done|completed|finished)\b/i;
+  return names.length > 0 && reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => subject.test(sentence) && yes.test(sentence) && !wrong.test(sentence));
+}
+export function gradeIntegratorRelatedWork(value: unknown) {
+  const c = checks(); const o = relatedObservation(value);
+  c.add("actual-case-relationships-and-principal-denial", !!o, "Requires public case/issue relationship records and a real responding-principal access probe.");
+  if (!o) return c.result();
+  const original = o.state.case as CaseRecord; const followup = o.state.followup as Entity;
+  const primary = o.tasks.find(row => row.id === original?.workIssueId); const linked = o.tasks.find(row => row.id === followup?.id);
+  c.add("original-primary-and-real-linked-followup", !!primary && !!linked && o.caseDetail.id === original?.id && o.caseDetail.workIssueId === primary.id
+    && o.caseDetail.issueLinks.some(link => link.issueId === linked.id && link.role === "reference")
+    && o.reverseLinks[primary.id]?.some(row => row.case.id === original.id && row.case.workIssueId === primary.id) && o.reverseLinks[linked.id]?.some(row => row.case.id === original.id && row.role === "reference"),
+  "The actual original primary and explicitly linked follow-up retain supported forward and reverse case relationships.");
+  c.add("accessible-work-state-and-links", !!primary && !!linked && hasLink(o, "issues", primary) && hasLink(o, "issues", linked)
+    && primary.status === "done" && linked.status === "backlog" && /\b(?:primary|main)\b/i.test(o.reply) && /\b(?:follow.up|linked|reference)\b/i.test(o.reply)
+    && reportsTaskState(o.reply, primary, "done") && reportsTaskState(o.reply, linked, "backlog")
+    && !/\b(?:all (?:case )?work (?:is )?(?:done|complete)|follow.up (?:is )?(?:done|completed))\b/i.test(o.reply),
+  "Chat links both actual tasks, distinguishes their roles, and preserves completed primary versus backlog follow-up rather than declaring all work done.");
+  const unrelated = o.state.unrelated as Entity; const privateTask = o.state.privateTask as Entity;
+  c.add("no-name-inference-or-private-disclosure", [403, 404].includes(o.deniedStatus) && !!unrelated && !!privateTask
+    && !hasLink(o, "issues", unrelated) && !hasLink(o, "issues", privateTask)
+    && !o.reply.includes(String(o.state.privateMarker)) && !o.reply.includes(privateTask.title ?? "missing-private-title")
+    && !o.caseDetail.issueLinks.some(link => link.issueId === unrelated.id || link.issueId === privateTask.id),
+  "The CEO's own genuine key cannot read the foreign private task; similar names do not create relationships or disclose inaccessible content.");
+  c.add("read-only-case-and-work", o.caseDetail.handoffVersion === original.handoffVersion && o.caseDetail.status === original.status
+    && !o.activity.some(event => event.agentId === o.leadId && /^(?:case\.|issue\.(?:created|updated))/.test(event.action ?? "")),
+  "Listing work preserves actual case/task state and cannot substitute new relationships for reading them.");
+  return c.result();
+}
+export const integratorRelatedWorkCases = [integratorDefinition("cases", "cases.related-work", async ctx => {
+  const settings = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); resources.set(ctx, { experimental: settings.enableCases });
+  try {
+    await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true });
+    const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx);
+    for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
+      await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
+    const kickoff = await ctx.input.api.post<HandoffCaseResult>(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, {
+      idempotencyKey: `related-work-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Mobile checkout customer case", projectId: p.id, fields: { symptom: "Synthetic mobile checkout fails" } } },
+      toAgentId: worker.id, phase: "investigation", instructions: "Read the saved brief, record that the synthetic mobile checkout diagnosis needs a later follow-up, and mark this initial investigation task done. Do not close the case, hand it off, create tasks or change other records.",
+      workflow_document_text: "# Investigation\nMorgan completes the initial investigation task. The case remains open while a separate linked follow-up is pending. Task completion alone is not a terminal case handoff.",
+      initialDocuments: [{ key: "brief", title: "Customer brief", body: "Synthetic mobile checkout issue. Initial investigation records observations; later follow-up remains separate." }],
+    });
+    const row = await ctx.input.api.get<CaseRecord>(`/api/cases/${kickoff.caseId}`);
+    await pollUntil({ label: "real case prerequisite primary completion", deadlineAt: Date.now() + 180_000, intervalMs: 1000,
+      load: async () => ({ task: await ctx.input.api.get<Entity>(`/api/issues/${kickoff.taskId}`), runs: await ctx.allRuns() }),
+      accept: value => value.task.status === "done" && value.runs.length === 1 && value.runs[0]?.status === "succeeded",
+      reject: value => value.runs.length > 1 ? "Case prerequisite exceeded its one real investigation run" : undefined });
+    const followup = await task(ctx, "Mobile checkout follow-up", p.id, { description: `Follow-up for case ${row.identifier}; remains backlog until assigned.` });
+    await ctx.input.api.post(`/api/cases/${row.id}/links`, { issueId: followup.id, role: "reference" });
+    const unrelated = await task(ctx, "Mobile checkout follow-up", p.id, { description: "Similar title, but explicitly unrelated and not linked to any case." });
+    const foreign = await ctx.input.api.post<Entity>("/api/companies", { name: `Private unrelated ${ctx.input.nonce}` }); resources.get(ctx)!.foreignCompanyId = foreign.id;
+    const privateMarker = `PRIVATE-CUSTOMER-${ctx.input.nonce}`;
+    const privateTask = await ctx.input.api.post<Entity>(`/api/companies/${foreign.id}/issues`, { title: `Confidential customer ${ctx.input.nonce}`, description: privateMarker, status: "backlog" });
+    if (!ctx.input.privateSecrets) throw new Error("Principal access probe requires private redaction");
+    const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: "case-read-boundary" });
+    ctx.input.privateSecrets.push(key.token);
+    let deniedStatus: number;
+    try { const response = await fetch(new URL(`/api/issues/${privateTask.id}`, ctx.input.api.baseURL), { headers: { Authorization: `Bearer ${key.token}` }, redirect: "error", signal: AbortSignal.timeout(20_000) }); deniedStatus = response.status; await response.body?.cancel(); }
+    finally { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); }
+    if (![403, 404].includes(deniedStatus)) throw new Error("Actual responding principal can read supposedly inaccessible task");
+    return { project: p, worker: { id: worker.id }, case: await ctx.input.api.get<CaseRecord>(`/api/cases/${row.id}`), followup, unrelated, privateTask, privateMarker, deniedStatus };
+  } catch (error) { await cleanup(ctx); throw error; }
+}, observeRelated, gradeIntegratorRelatedWork, async (ctx, _state, value) => {
+  const o = relatedObservation(value); if (!o) throw new Error("Missing real case relationships");
+  const source = ctx.input.page.url();
+  for (const id of [o.caseDetail.workIssueId!, (o.state.followup as Entity).id]) {
+    const task = o.tasks.find(row => row.id === id); const href = task && (resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues"));
+    if (!task || !href) throw new Error("Missing accessible case task link");
+    await ctx.input.page.goto(new URL(href, o.origin).href);
+    await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible();
+    const reverse = ctx.input.page.locator(`a[href$="/cases/${o.caseDetail.identifier}"], a[href$="/cases/${o.caseDetail.id}"]`).first();
+    await expect(reverse).toBeVisible(); await reverse.click();
+    await expect(ctx.input.page.getByText(o.caseDetail.title, { exact: true }).first()).toBeVisible();
+    await ctx.input.evidence(`chat-case-reverse-link-${id}.json`, { caseId: o.caseDetail.id, issueId: id, url: ctx.input.page.url() });
+  }
+  await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+  await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${o.caseDetail.identifier}`, o.caseDetail.title);
+}, { sourceFiles: ["chat-catalog-domains/integrator-case-related.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"],
+  expectedRunCount: 2, attemptTimeoutMs: 540_000, cleanup,
+})];