diff --git a/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.test.ts index 9fe7157b69a..c9107a9fd62 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradeAgentFeedback } from "./internal-agent-feedback.js"; +import { commentaryReaderAttribution, gradeAgentFeedback } from "./internal-agent-feedback.js"; function valid() { const t = { id: "task", title: "Feedback prerequisite" }; const hidden = { id: "hidden", title: "Restricted source" }; const entry = (id: string, kind: string) => ({ id, kind, preview: `${kind} about tools`, runId: "morgan-run", agent: { id: "morgan", name: "Morgan" }, task: t }); @@ -19,3 +19,16 @@ describe("human-only agent commentary calibration", () => { it("rejects foreign commentary links and changed submitted content", () => { const o = valid(); o.links = ["https://foreign.test/RUN/activity/commentary"]; expect(gradeAgentFeedback(o).passed).toBe(false); const changed = valid(); changed.entries[0]!.preview = "Replaced"; expect(gradeAgentFeedback(changed).passed).toBe(false); }); it("rejects a new CEO submission instead of listing", () => { const o = valid(); o.activity.push({ action: "agent.commentary_submitted", agentId: "ceo" }); expect(gradeAgentFeedback(o).passed).toBe(false); }); }); +describe("canonical feedback reader attribution", () => { + const actual = () => ({ readerId: "casey", sourceId: "source", leadId: "lead", comments: [{ id: "comment", authorUserId: "casey" }], + runs: [{ id: "source-run", agentId: "lead", responsibleUserId: "casey", contextSnapshot: { issueId: "source", commentId: "comment" } }] }); + it("accepts exact real canonical human/comment/run attribution", () => expect(commentaryReaderAttribution(actual()).sourceRunIds).toEqual(["source-run"])); + it.each(["human", "responsible", "source", "comment"]) ("rejects wrong %s attribution", kind => { + const o = actual(); + if (kind === "human") o.comments[0]!.authorUserId = "owner"; + if (kind === "responsible") o.runs[0]!.responsibleUserId = "owner"; + if (kind === "source") o.runs[0]!.contextSnapshot.issueId = "private-source"; + if (kind === "comment") o.runs[0]!.contextSnapshot.commentId = "different-comment"; + expect(() => commentaryReaderAttribution(o)).toThrow(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.ts b/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.ts index 503f12e78e0..946ea711c5d 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.ts @@ -5,6 +5,7 @@ import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { mainPerson } from "./internal-privacy.js"; import { checks, entity, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js"; +import { installCommentaryReader } from "./internal-commentary-reader.js"; interface Feedback { id: string; kind: string; preview: string; runId: string; agent: { id: string; name: string }; task: { id: string; title: string } | null; body?: string } interface FeedbackObservation extends InternalObservation { entries: Feedback[]; hiddenEntry?: Feedback; hiddenDetailStatus: number; agentReadStatus: number; privateTaskStatus: number } function commentaryHref(o: InternalObservation) { @@ -38,8 +39,9 @@ export function gradeAgentFeedback(value: unknown) { } function agentFeedback(): ChatCatalogCase { let casey: Awaited> | undefined; let key: { id: string; token: string } | undefined; + let readerSession: Awaited> | undefined; const base = makeDefinition("activity.agent-feedback", async ctx => { - await mainPerson(ctx); const browser = ctx.input.page.context().browser(); + const ownerId = await mainPerson(ctx); const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Feedback privacy needs a distinct real human and attempt-private redaction"); casey = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); try { @@ -50,20 +52,18 @@ function agentFeedback(): ChatCatalogCase { for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker, privateWorker].map(agent => ({ scopeType: "agent", scopeId: agent.id }))]) await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); const p = await project(ctx); const privateMarker = `RESTRICTED-COMMENTARY-${ctx.input.nonce}`; - const response = await casey.request(`${root}/projects`, "POST", { name: "Casey's restricted launch", visibility: "private" }); - if (!response.ok) { await response.body?.cancel(); throw new Error("Independent person's restricted project did not persist"); } - const privateProject = await response.json() as Entity; - const grant = await casey.request(`/api/projects/${privateProject.id}/access-members`, "POST", { subjectType: "agent", subjectId: privateWorker.id }); - if (!grant.ok) { await grant.body?.cancel(); throw new Error("Private source worker did not receive actual project access"); } await grant.body?.cancel(); - const hiddenResponse = await casey.request(`${root}/issues`, "POST", { title: "Restricted feedback prerequisite", projectId: privateProject.id, status: "todo", priority: "medium", assigneeAgentId: privateWorker.id, + // The bootstrap person owns the restricted source. Casey is the actual + // nonadmin canonical reader, never its owner or an inherited grant. + const privateProject = await ctx.input.api.post(`${root}/projects`, { name: "Restricted launch source", visibility: "private" }); + if (privateProject.privacyOwnerUserId !== ownerId) throw new Error("Private feedback source does not retain its actual separate owner"); + await ctx.input.api.post(`/api/projects/${privateProject.id}/access-members`, { subjectType: "agent", subjectId: privateWorker.id }); + const hidden = await ctx.input.api.post(`${root}/issues`, { title: "Restricted feedback prerequisite", projectId: privateProject.id, status: "todo", priority: "medium", assigneeAgentId: privateWorker.id, description: `Synthetic qualification prerequisite. Submit exactly one sanitized suggestion through the bound submit_suggestion tool, with body: 'Keep run context discoverable (${privateMarker}).' Use a stable idempotency key. Do not expose private task body or quote prompts in feedback. Finish this small prerequisite task after the real tool acknowledgement; no other work is requested.` }); - if (!hiddenResponse.ok) { await hiddenResponse.body?.cancel(); throw new Error("Private feedback prerequisite task was not admitted"); } const hidden = await hiddenResponse.json() as Entity; const t = await task(ctx, "Synthetic agent feedback prerequisite", p.id, { status: "todo", assigneeAgentId: worker.id, description: "Synthetic qualification prerequisite. Submit one complaint through the bound submit_complaint tool: 'Tool discovery is hard to scan in a long run.' Submit a separate suggestion through submit_suggestion: 'Keep a compact index of available tools.' These are separate synthetic fixture incidents. Use separate stable idempotency keys; submit only once each. Finish this prerequisite task after the real acknowledgements; do not browse or summarize other agents' feedback." }); const saved = await pollUntil({ label: "Genuine run-bound prior agent suggestions and complaints", deadlineAt: Date.now() + 240_000, load: async () => { const entries = await ctx.input.api.get<{ items: Feedback[] }>(`${root}/agent-commentary?limit=100`); - const privateRead = await casey!.request(`${root}/agent-commentary?limit=100`); if (!privateRead.ok) { await privateRead.body?.cancel(); throw new Error("Authorized private source owner cannot inspect fixture feedback"); } - return { entries: entries.items, privateEntries: (await privateRead.json() as { items: Feedback[] }).items, runs: await ctx.allRuns() }; }, + return { entries: entries.items, privateEntries: entries.items, runs: await ctx.allRuns() }; }, accept: value => value.runs.length === 2 && value.runs.every(row => row.status === "succeeded") && value.entries.filter(row => row.agent.id === worker.id && row.task?.id === t.id).length === 2 && value.privateEntries.some(row => row.agent.id === privateWorker.id && row.task?.id === hidden.id && row.preview.includes(privateMarker)), @@ -71,11 +71,26 @@ function agentFeedback(): ChatCatalogCase { const hiddenEntry = saved.privateEntries.find(row => row.agent.id === privateWorker.id && row.task?.id === hidden.id)!; key = await ctx.input.api.postSensitive(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `commentary-read-control-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key!.token); await ctx.input.evidence("chat-catalog-genuine-feedback-prerequisites.json", { entries: saved.entries, hiddenEntry, runIds: saved.runs.map(row => row.id) }); - return { project: p, task: t, worker, privateWorker, privateTask: hidden, privateMarker, initialEntries: saved.entries.filter(row => row.agent.id === worker.id), hiddenEntry }; - } catch (error) { await casey.dispose(); casey = undefined; throw error; } + readerSession = await installCommentaryReader(ctx, casey, ownerId); + const visible = await ctx.input.api.get<{ items: Feedback[] }>(`${root}/agent-commentary?limit=100`); + const statuses = await Promise.all([`/api/issues/${hidden.id}`, `${root}/agent-commentary/${hiddenEntry.id}`].map(async path => { + const response = await casey!.request(path); await response.body?.cancel(); return response.status; + })); + const initialEntries = saved.entries.filter(row => row.agent.id === worker.id); + if (visible.items.length !== 2 || !initialEntries.every(row => visible.items.some(item => item.id === row.id)) + || !statuses.every(status => [403, 404].includes(status))) throw new Error("Canonical nonadmin reader does not have the intended visible-feedback/private-source boundary"); + await ctx.input.evidence("chat-catalog-commentary-reader-prerequisite.json", { ...readerSession.receipt, + visibleEntryIds: visible.items.map(row => row.id), privateTaskStatus: statuses[0], privateFeedbackStatus: statuses[1] }); + return { project: p, task: t, worker, privateWorker, privateTask: hidden, privateMarker, initialEntries, hiddenEntry, readerIdentity: readerSession.receipt }; + } catch (error) { + try { await readerSession?.restore(); readerSession = undefined; + if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } + } finally { await casey.dispose(); casey = undefined; } + throw error; + } }, gradeAgentFeedback); return { ...base, authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 600_000, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-agent-feedback.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts"], + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-agent-feedback.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-commentary-reader.ts", "chat-run-evidence-observer.ts"], async observe(ctx, state): Promise { if (!key) throw new Error("Feedback read probe was not created during setup"); const o = await observeInternal(ctx, state); const root = `/api/companies/${o.companyId}`; const hiddenEntry = state.hiddenEntry as Feedback; @@ -87,6 +102,12 @@ function agentFeedback(): ChatCatalogCase { const [entries, hiddenDetailStatus, privateTaskStatus, agentReadStatus] = await Promise.all([ ctx.input.api.get<{ items: Feedback[] }>(`${root}/agent-commentary?limit=100`), status(`${root}/agent-commentary/${hiddenEntry.id}`), status(`/api/issues/${entity(state, "privateTask").id}`), status(`${root}/agent-commentary?limit=100`, true), ]); + const source = ctx.issue(); + if (!source) throw new Error("Canonical commentary source is missing"); + const comments = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + const attribution = commentaryReaderAttribution({ readerId: (state.readerIdentity as { readerId: string }).readerId, + sourceId: source.id, leadId: o.leadId, comments, runs: o.runs }); + await ctx.input.evidence("chat-catalog-commentary-canonical-attribution.json", attribution); return { ...o, entries: entries.items, hiddenEntry, hiddenDetailStatus, privateTaskStatus, agentReadStatus }; }, async verifyResult(ctx, _state, value) { @@ -107,7 +128,22 @@ function agentFeedback(): ChatCatalogCase { await ctx.input.evidence("chat-catalog-authorized-human-commentary-opened.json", { url: ctx.input.page.url(), entryIds: (o.state.initialEntries as Feedback[]).map(row => row.id), hiddenExcluded: true }); } finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, - async cleanup(ctx) { try { if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } } finally { await casey?.dispose(); casey = undefined; } }, + async cleanup(ctx) { try { + await readerSession?.restore(); readerSession = undefined; + if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } + } finally { await casey?.dispose(); casey = undefined; } }, }; } export const agentFeedbackCases: readonly ChatCatalogCase[] = [agentFeedback()]; + +export function commentaryReaderAttribution(input: { readerId: string; sourceId: string; leadId: string; + comments: Array<{ id: string; authorUserId?: string | null; authorAgentId?: string | null }>; + runs: Array<{ id: string; agentId: string; responsibleUserId?: string | null; contextSnapshot?: Record }> }) { + const human = input.comments.filter(row => !row.authorAgentId); + if (human.length !== 1 || human[0]!.authorUserId !== input.readerId) throw new Error("Canonical feedback comment is not attributed to the actual reader"); + const sourceRuns = input.runs.filter(row => row.agentId === input.leadId); + if (sourceRuns.some(row => row.responsibleUserId !== input.readerId + || row.contextSnapshot?.issueId !== input.sourceId || row.contextSnapshot?.commentId !== human[0]!.id)) + throw new Error("Canonical feedback execution does not retain its real reader/comment/source attribution"); + return { readerId: input.readerId, sourceIssueId: input.sourceId, commentId: human[0]!.id, sourceRunIds: sourceRuns.map(row => row.id) }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.test.ts new file mode 100644 index 00000000000..8ca74b2b1b8 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.test.ts @@ -0,0 +1,67 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const install = vi.hoisted(() => vi.fn(async (context: { restoreOwner(): void }) => context.restoreOwner())); +vi.mock("./identity-auth.js", () => ({ installIdentitySession: install })); +vi.mock("../api.js", () => ({ RunnerApi: class { + baseURL = "http://127.0.0.1:3100"; + setBrowserSession() {} + async getRunObservation() { return []; } + async get() { return {}; } +} })); +import { installCommentaryReader } from "./internal-commentary-reader.js"; + +function fixture() { + let apiUser = "owner"; let browserUser = "owner"; + let source: { id: string; companyId: string; conversationUserId: string; conversationAgentId: string } | undefined; + const state = { browserOverride: undefined as string | undefined, sourceOverride: undefined as string | undefined }; + const response = { ok: true, json: async () => ({ user: { id: state.browserOverride ?? browserUser } }), body: { cancel: vi.fn(async () => undefined) } }; + vi.stubGlobal("fetch", vi.fn(async () => response)); + const context = { cookies: vi.fn(async () => [{ name: "session", value: browserUser === "owner" ? "owned-test-cookie" : "reader-test-cookie" }]), + request: { get: vi.fn(async () => response) }, restoreOwner() { browserUser = "owner"; } }; + const api = { baseURL: "http://127.0.0.1:3100", get: vi.fn(async () => ({ user: { id: apiUser } })), + setBrowserSession: vi.fn(() => { apiUser = "owner"; }), + post: vi.fn(async () => source = { id: "source", companyId: "11111111-1111-4111-8111-111111111111", conversationUserId: state.sourceOverride ?? apiUser, conversationAgentId: "lead" }) }; + const evidence = vi.fn(async () => undefined); + const page = { context: () => context, reload: vi.fn(async () => undefined) }; + const ctx = { scenario: { id: "activity.agent-feedback" }, input: { api, page, evidence, fixtures: { company: { id: "11111111-1111-4111-8111-111111111111" }, agent: { id: "lead" } } }, + refreshIssue: vi.fn(async () => undefined), issue: () => source } as unknown as ChatCatalogContext; + const person = { id: "casey", installSessionOn: vi.fn(async () => { browserUser = "casey"; }), + bindApiSession: vi.fn(() => { apiUser = "casey"; }) } as unknown as Parameters[1]; + return { ctx, person, api, context, state, evidence, response, current: () => ({ apiUser, browserUser }) }; +} +beforeEach(() => { install.mockClear(); }); +afterEach(() => { vi.unstubAllGlobals(); }); +describe("real commentary reader identity lifetime", () => { + it("aligns real API/browser/conversation identities and restores the original owner only once", async () => { + const f = fixture(); const selected = await installCommentaryReader(f.ctx, f.person, "owner"); + expect(f.current()).toEqual({ apiUser: "casey", browserUser: "casey" }); + expect(selected.receipt).toEqual({ ownerId: "owner", readerId: "casey", apiUserId: "casey", browserUserId: "casey", sourceIssueId: "source", sourceConversationUserId: "casey" }); + expect(JSON.stringify(f.evidence.mock.calls)).not.toMatch(/owned-test-cookie|session=/); + expect(fetch).toHaveBeenCalledWith(new URL("/api/auth/get-session", f.api.baseURL), expect.objectContaining({ headers: { Cookie: "session=reader-test-cookie", Origin: f.api.baseURL } })); + await selected.restore(); await selected.restore(); + expect(f.current()).toEqual({ apiUser: "owner", browserUser: "owner" }); + expect(install).toHaveBeenCalledOnce(); expect(f.response.body.cancel).toHaveBeenCalledOnce(); + }); + it("restores genuine owner sessions after a browser/API identity mismatch", async () => { + const f = fixture(); f.state.browserOverride = "different-reader"; + await expect(installCommentaryReader(f.ctx, f.person, "owner")).rejects.toThrow("same real reader"); + expect(f.current()).toEqual({ apiUser: "owner", browserUser: "owner" }); + expect(f.evidence).not.toHaveBeenCalled(); + }); + it("rejects a conversation owned by a different person, retaining no false identity receipt", async () => { + const f = fixture(); f.state.sourceOverride = "owner"; + await expect(installCommentaryReader(f.ctx, f.person, "owner")).rejects.toThrow("actual reader"); + expect(f.current()).toEqual({ apiUser: "owner", browserUser: "owner" }); + expect(f.evidence).not.toHaveBeenCalled(); + }); + it("rejects substituting the existing private owner for a distinct excluded reader", async () => { + const f = fixture(); f.person.id = "owner"; + await expect(installCommentaryReader(f.ctx, f.person, "owner")).rejects.toThrow("distinct real owner and reader"); + expect(f.person.installSessionOn).not.toHaveBeenCalled(); expect(install).not.toHaveBeenCalled(); + }); + it("requires a genuine restorable owner browser session before switching", async () => { + const f = fixture(); f.context.cookies.mockResolvedValue([]); + await expect(installCommentaryReader(f.ctx, f.person, "owner")).rejects.toThrow("genuine browser session"); + expect(f.person.installSessionOn).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.ts b/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.ts new file mode 100644 index 00000000000..84021125da8 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-commentary-reader.ts @@ -0,0 +1,65 @@ +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { installIdentitySession, type createAuthenticatedPerson } from "./identity-auth.js"; +import { RunnerApi } from "../api.js"; +import { createOwnedChatRunEvidenceObserver } from "../chat-run-evidence-observer.js"; + +type Person = Awaited>; +/** One case's real person transition. Session material remains in this closure; + * all retained receipts contain public user/conversation identities only. */ +export async function installCommentaryReader(ctx: ChatCatalogContext, person: Person, ownerId: string) { + const { api, page } = ctx.input; + if (ctx.scenario.id !== "activity.agent-feedback" || ctx.input.ownedRunEvidenceObserver) + throw new Error("Commentary reader installation requires its exact unbound scenario"); + const original = await api.get<{ user?: { id: string } }>("/api/auth/get-session"); + if (original.user?.id !== ownerId || person.id === ownerId) throw new Error("Commentary needs distinct real owner and reader sessions"); + const cookies = await page.context().cookies(api.baseURL); + if (!cookies.length) throw new Error("Commentary owner has no genuine browser session to restore"); + const cookie = cookies.map(row => `${row.name}=${row.value}`).join("; "); + // A separate client freezes only the real owner's existing cookie. Its + // mutation methods never escape: the approved observer exposes GETs only. + const evidenceApi = new RunnerApi(page.context().request); + evidenceApi.setBrowserSession(cookie); + ctx.input.ownedRunEvidenceObserver = createOwnedChatRunEvidenceObserver(evidenceApi, ctx.input.fixtures.company.id); + let restored = false; + const restore = async () => { + if (restored) return; + await installIdentitySession(page.context(), api.baseURL, cookie); + api.setBrowserSession(cookie); + if ((await api.get<{ user?: { id: string } }>("/api/auth/get-session")).user?.id !== ownerId) + throw new Error("Commentary owner session restoration did not preserve its real identity"); + restored = true; + }; + try { + await person.installSessionOn(page.context()); + person.bindApiSession(api); + // Reload drops the previous person's cached React state as well as using + // the newly installed genuine server session for all following UI requests. + await page.reload({ waitUntil: "domcontentloaded" }); + const apiPerson = await api.get<{ user?: { id: string } }>("/api/auth/get-session"); + // Independently authenticate the actual browser jar, without recording the + // auth DTO or cookie header in Playwright's API trace. + const browserCookie = (await page.context().cookies(api.baseURL)).map(row => `${row.name}=${row.value}`).join("; "); + const response = await fetch(new URL("/api/auth/get-session", api.baseURL), { redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { Cookie: browserCookie, Origin: api.baseURL } }); + let browserPerson: { user?: { id: string } }; + try { + if (!response.ok) throw new Error("Commentary browser session could not be independently read"); + browserPerson = await response.json(); + } finally { await response.body?.cancel(); } + if (apiPerson.user?.id !== person.id || browserPerson.user?.id !== person.id) + throw new Error("Commentary API and browser do not represent the same real reader"); + const source = await api.post<{ id: string; companyId: string; conversationUserId: string; conversationAgentId: string }>( + `/api/companies/${ctx.input.fixtures.company.id}/chats/${ctx.input.fixtures.agent.id}`); + await ctx.refreshIssue(); + if (source.companyId !== ctx.input.fixtures.company.id || source.conversationUserId !== person.id + || source.conversationAgentId !== ctx.input.fixtures.agent.id || ctx.issue()?.id !== source.id) + throw new Error("Canonical commentary conversation is not owned by its actual reader"); + const receipt = { ownerId, readerId: person.id, apiUserId: apiPerson.user.id, browserUserId: browserPerson.user.id, + sourceIssueId: source.id, sourceConversationUserId: source.conversationUserId }; + await ctx.input.evidence("chat-catalog-commentary-reader-identity.json", receipt); + return { receipt, restore }; + } catch (error) { + await restore(); + throw error; + } +} diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index b3809592a19..c329d23cec6 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -11,6 +11,7 @@ import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCata import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js"; import { resolveChatCatalogQuery } from "./chat-query-bindings.js"; +import { chatRunEvidenceApi } from "./chat-run-evidence-observer.js"; /** The case owns product semantics; this flow owns the actual chat entry point and evidence lifecycle. */ export async function runChatCatalogFlow(context: ChatCatalogContext) { @@ -171,7 +172,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { await conversation(); if (state) await refresh(); const runs = await context.allRuns(); - await input.evidence("chat-catalog-runs.json", await Promise.all(runs.map(run => collectChatRunEvidence(input.api, run)))); + await input.evidence("chat-catalog-runs.json", await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run)))); } catch { await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }); if (!failed) throw new Error("Scenario evidence collection failed"); diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index d0807bf448b..93ed63ce508 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -26,6 +26,7 @@ import { runChatCompletionUpdate } from "./completion-update-flow.js"; import { findChatCatalogCase } from "./chat-catalog-registry.js"; import { CHAT_FEATURE_SUITE } from "./chat-feature-cases.js"; import { runChatFeatureFlow } from "./chat-feature-flow.js"; +import { chatRunEvidenceApi, readChatRuns, type OwnedChatRunEvidenceObserver } from "./chat-run-evidence-observer.js"; /** Hiring alone admits bounded, verified lifecycle notifications. Other suites keep their count contract. */ export function assertChatFlowRunCount(input: { @@ -361,6 +362,8 @@ export function assertChatReassignment(input: { } export interface ChatFlowInput { + /** Exact commentary privacy case only: independent GET-only owned run evidence. Never model/product authority. */ + ownedRunEvidenceObserver?: OwnedChatRunEvidenceObserver; /** Test-host-only owned transparent fault transport; never model state. */ archiveFaultTransport?: ArchiveFaultTransport; page: Page; @@ -396,14 +399,7 @@ export async function runChatFlow(input: ChatFlowInput) { const settings = await api.get>( "/api/instance/settings/experimental", ); - const allRuns = async () => { - const rows = await api.getRunObservation( - `/api/companies/${f.company.id}/heartbeat-runs?limit=100`, - ); - return Promise.all( - rows.map((row) => api.getRunObservation(`/api/heartbeat-runs/${row.id}`)), - ); - }; + const allRuns = () => readChatRuns(input); if (catalogDescriptor?.journeyKind === "native-slack-thread") { let sourceId: string | undefined; await runChatFeatureFlow({ input, issue: () => issue, allRuns, @@ -1085,7 +1081,7 @@ export async function runChatFlow(input: ChatFlowInput) { comments: await comments(), activity: await api.get(`/api/issues/${issue!.id}/activity`), runEvidence: await Promise.all( - runs.map((run) => collectChatRunEvidence(api, run)), + runs.map((run) => collectChatRunEvidence(chatRunEvidenceApi(input), run)), ), }); await input.capture( diff --git a/tests/runner-e2e/chat-run-evidence-observer.test.ts b/tests/runner-e2e/chat-run-evidence-observer.test.ts new file mode 100644 index 00000000000..c77ce425073 --- /dev/null +++ b/tests/runner-e2e/chat-run-evidence-observer.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from "vitest"; +import { chatRunEvidenceApi, createOwnedChatRunEvidenceObserver, readChatRuns } from "./chat-run-evidence-observer.js"; +import type { ChatFlowInput } from "./chat-flow.js"; +const companyId = "11111111-1111-4111-8111-111111111111", runId = "22222222-2222-4222-8222-222222222222", foreign = "33333333-3333-4333-8333-333333333333"; +function fixture() { + const row = { id: runId, companyId, agentId: "worker", status: "succeeded" }; + const owner = { baseURL: "http://127.0.0.1:3100", get: vi.fn(async () => ({ content: "actual owned log" })), + getRunObservation: vi.fn(async (path: string): Promise => path.includes("/companies/") ? [row] : path.includes("/events?") ? [] : row) }; + const observer = createOwnedChatRunEvidenceObserver(owner as Parameters[0], companyId); + const actor = { baseURL: owner.baseURL, get: vi.fn(async () => { throw Object.assign(new Error("Private source denied"), { status: 403 }); }), getRunObservation: vi.fn(async (path: string): Promise => path.includes("/companies/") ? [row] : row) }; + const input = { api: actor, fixtures: { company: { id: companyId } }, execution: { suite: { id: "chat-features" }, task: { id: "activity.agent-feedback" } }, ownedRunEvidenceObserver: observer } as unknown as ChatFlowInput; + return { owner, observer, row, actor, input }; +} +describe("exact commentary case independent run evidence", () => { + it("uses the privately frozen owner only for GET evidence, preserving the actual product actor", async () => { + const f = fixture(); expect(await readChatRuns(f.input)).toEqual([f.row]); + await f.observer.getRunObservation(`/api/heartbeat-runs/${runId}/events?afterSeq=0&limit=1000`); + await f.observer.get(`/api/heartbeat-runs/${runId}/log?limitBytes=1048576`); + expect(f.owner.get).toHaveBeenCalledExactlyOnceWith(`/api/heartbeat-runs/${runId}/log?limitBytes=1048576`); + expect(f.actor.getRunObservation).not.toHaveBeenCalled(); expect(f.input.api).toBe(f.actor); + await expect(f.input.api.get(`/api/issues/${foreign}`)).rejects.toMatchObject({ status: 403 }); + expect(Object.keys(f.observer).sort()).toEqual(["companyId", "origin", "listRuns", "get", "getRunObservation"].sort()); + expect(Object.isFrozen(f.observer)).toBe(true); + }); + it("keeps ordinary/default run collection on the unchanged product API", async () => { + const f = fixture(); delete f.input.ownedRunEvidenceObserver; + expect(chatRunEvidenceApi(f.input)).toBe(f.actor); expect(await readChatRuns(f.input)).toEqual([f.row]); + expect(f.owner.getRunObservation).not.toHaveBeenCalled(); + }); + it.each(["scenario", "company", "origin"]) ("rejects an observer borrowed into another %s", kind => { + const f = fixture(); + if (kind === "scenario") f.input.execution.task.id = "activity.work-timeline"; + if (kind === "company") f.input.fixtures.company.id = foreign; + if (kind === "origin") Object.assign(f.actor, { baseURL: "https://other.test" }); + expect(() => chatRunEvidenceApi(f.input)).toThrow("exact approved scenario/company/origin"); + }); + it("rejects a run not independently listed in the owned company", async () => { + const f = fixture(); await expect(f.observer.get(`/api/heartbeat-runs/${runId}`)).rejects.toThrow("independently listed"); + await f.observer.listRuns(); await expect(f.observer.get(`/api/heartbeat-runs/${foreign}`)).rejects.toThrow("independently listed"); + }); + it.each(["https://foreign.test/api/heartbeat-runs/", "/api/issues/", "/api/heartbeat-runs/"]) ("rejects unauthorized origin/path %s", async prefix => { + const f = fixture(); await f.observer.listRuns(); + const path = prefix + runId + (prefix === "/api/heartbeat-runs/" ? "/cancel" : ""); + await expect(f.observer.get(path)).rejects.toThrow(); + }); + it("rejects foreign/duplicate company listings and mismatched detail identity", async () => { + const f = fixture(); f.row.companyId = foreign; await expect(f.observer.listRuns()).rejects.toThrow("foreign"); + const duplicate = fixture(); duplicate.owner.getRunObservation.mockResolvedValue([duplicate.row, duplicate.row]); + await expect(duplicate.observer.listRuns()).rejects.toThrow("duplicate"); + const wrong = fixture(); await wrong.observer.listRuns(); wrong.row.companyId = foreign; + await expect(wrong.observer.get(`/api/heartbeat-runs/${runId}`)).rejects.toThrow("owned list identity"); + }); + it.each(["/events?afterSeq=0&limit=5&mutation=1", "/log?limitBytes=99999999", "?includeSecrets=1", "/events?afterSeq=0&limit=5#fragment"])("rejects unauthorized evidence query %s", async suffix => { + const f = fixture(); await f.observer.listRuns(); await expect(f.observer.get(`/api/heartbeat-runs/${runId}${suffix}`)).rejects.toThrow(); + }); +}); diff --git a/tests/runner-e2e/chat-run-evidence-observer.ts b/tests/runner-e2e/chat-run-evidence-observer.ts new file mode 100644 index 00000000000..3b893ab89e6 --- /dev/null +++ b/tests/runner-e2e/chat-run-evidence-observer.ts @@ -0,0 +1,64 @@ +import type { RunnerApi } from "./api.js"; +import type { ChatFlowInput, ChatRun } from "./chat-flow.js"; + +export interface OwnedChatRunEvidenceObserver { + readonly companyId: string; + readonly origin: string; + listRuns(): Promise; + get(path: string): Promise; + getRunObservation(path: string): Promise; +} +const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i; +/** The caller freezes a genuine owner's separate authenticated client before + * changing the product actor. The returned interface exposes no mutation API. */ +export function createOwnedChatRunEvidenceObserver(api: Pick, companyId: string): OwnedChatRunEvidenceObserver { + if (!uuid.test(companyId)) throw new Error("Run evidence observer requires the owned company identity"); + const origin = new URL(api.baseURL).origin; + if (origin !== api.baseURL) throw new Error("Run evidence observer requires an exact API origin"); + const known = new Set(); + async function read(path: string): Promise { + const url = new URL(path, origin); + if (url.origin !== origin || url.username || url.password || url.hash) throw new Error("Run evidence observer escaped its origin"); + const match = /^\/api\/heartbeat-runs\/([^/]+)(?:\/(events|log))?$/.exec(url.pathname); + if (!match || !uuid.test(match[1]!) || !known.has(match[1]!)) throw new Error("Run evidence requires an independently listed owned run"); + const kind = match[2]; + const keys = [...url.searchParams.keys()]; + const allowed = !kind ? !url.search : kind === "events" + ? keys.length === 2 && keys.every(key => ["afterSeq", "limit"].includes(key)) && /^\d+$/.test(url.searchParams.get("afterSeq") ?? "") && /^[1-9]\d*$/.test(url.searchParams.get("limit") ?? "") + : keys.length === 1 && url.searchParams.get("limitBytes") === "1048576"; + if (!allowed) throw new Error("Run evidence observer rejected an unauthorized path or query"); + const value = kind === "log" ? await api.get(url.pathname + url.search) : await api.getRunObservation(url.pathname + url.search); + if (!kind) { + const row = value as ChatRun; + if (row.id !== match[1] || row.companyId !== companyId) throw new Error("Run evidence detail does not match its owned list identity"); + } + return value; + } + return Object.freeze({ companyId, origin, get: read, getRunObservation: read, + async listRuns() { + const rows = await api.getRunObservation(`/api/companies/${companyId}/heartbeat-runs?limit=100`); + if (!Array.isArray(rows) || rows.some(row => !uuid.test(row.id) || row.companyId !== companyId) || new Set(rows.map(row => row.id)).size !== rows.length) + throw new Error("Run evidence list contains foreign, invalid or duplicate run identities"); + known.clear(); for (const row of rows) known.add(row.id); + return Promise.all(rows.map(row => read(`/api/heartbeat-runs/${row.id}`))); + }, + }); +} + +/** This exceptional independent observer belongs to precisely one privacy case. + * Ordinary product calls always continue using input.api's actual human actor. */ +export function chatRunEvidenceApi(input: Pick): Pick { + const observer = input.ownedRunEvidenceObserver; + if (!observer) return input.api; + if (input.execution.suite.id !== "chat-features" || input.execution.task.id !== "activity.agent-feedback" + || observer.companyId !== input.fixtures.company.id || observer.origin !== input.api.baseURL) + throw new Error("Independent run evidence observer is outside its exact approved scenario/company/origin"); + return observer; +} + +export async function readChatRuns(input: Pick): Promise { + const api = chatRunEvidenceApi(input); + if (input.ownedRunEvidenceObserver) return input.ownedRunEvidenceObserver.listRuns(); + const rows = await api.getRunObservation(`/api/companies/${input.fixtures.company.id}/heartbeat-runs?limit=100`); + return Promise.all(rows.map(row => api.getRunObservation(`/api/heartbeat-runs/${row.id}`))); +}