diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 4f1f3b95753..b9997ea75d0 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1999,6 +1999,7 @@ "scenarioId": "companies.agent-observability", "sourceFiles": [ "chat-catalog-domains/identity-observability.ts", + "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", @@ -2007,7 +2008,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.agent-observability.structural.v3", + "oracleVersion": "paperclip.chat-catalog.agent-observability.structural.v4", "expectedRunCount": 3, "expectedHumanMessageCount": 3, "turnTimeoutMs": 120000, @@ -3454,7 +3455,7 @@ "chat-catalog-domains/internal-inbox-search.ts", "chat-scenarios/inbox-search.yaml" ], - "oracleVersion": "paperclip.chat-catalog.internal.show-my-work.v4", + "oracleVersion": "paperclip.chat-catalog.internal.show-my-work.v5", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -3559,9 +3560,10 @@ "chat-catalog-domains/privileged-support.ts", "chat-catalog-domains/turn-response-card.ts", "chat-catalog-domains/privileged-instance.ts", + "chat-rendered-links.ts", "chat-scenarios/instance-operations.yaml" ], - "oracleVersion": "paperclip.chat-catalog.privileged.restart-service.v2", + "oracleVersion": "paperclip.chat-catalog.privileged.restart-service.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts index 64a187697e6..0fe3cb3e1d1 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts @@ -1,14 +1,16 @@ import { describe, expect, it, vi } from "vitest"; -import { gradeObservability } from "./identity-observability.js"; +import { gradeObservability, readSavedObservabilityStatus } from "./identity-observability.js"; +import { savedCall } from "./api-read-receipts.test-fixtures.js"; vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn() })); import type { IdentityObservation } from "./identity-support.js"; -function valid(): IdentityObservation { return { scenarioId: "companies.agent-observability", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], humanQueries: [], +function valid(): IdentityObservation { return { scenarioId: "companies.agent-observability", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: ["/ACO/company/settings"], humanQueries: [], reply: "Configure optional trace export in company settings. First-party Telemetry and the local run log are separate data paths.", runs: [1, 2, 3].map(n => ({ id: `run-${n}`, companyId: "acorn", agentId: "ceo", status: "succeeded", runtimeMode: "native" })), state: { beforeSettings: null, afterReplySettings: null, beforeConfiguredCalls: 0, agentWriteStatus: 403, collectorUrl: "http://127.0.0.1:1234", publicSettingsContainsKey: false, saved: { settings: { companyId: "acorn", enabled: true, revision: "revision", destination: { provider: "otlp", endpointBaseUrl: "http://127.0.0.1:1234" }, headerNames: ["authorization"], credentialsConfigured: true } }, testReceipt: { status: "accepted", source: "control_plane", revision: "revision", signals: { logs: { status: "accepted" }, traces: { status: "accepted" } } }, collectorMatchedTestSignals: 2, - nativeExportMatched: true, enabledNativeRunCount: 2, disabledSaved: true, disabledNativeRunCount: 3, callsAtDisable: 7, callsAfterDisabledRun: 7, disabledCallsDidNotIncrease: true } }; } + nativeExportMatched: true, enabledNativeRunCount: 2, disabledSaved: true, disabledNativeRunCount: 3, callsAtDisable: 7, callsAfterDisabledRun: 7, disabledCallsDidNotIncrease: true, + companyPrefix: "ACO", enabledTurnReadSavedStatus: true, disabledTurnReadSavedStatus: true } }; } describe("actual optional agent trace delivery", () => { it("requires enabled human configuration, actual native export and disabled subsequent execution", () => { expect(gradeObservability(valid()).passed).toBe(true); expect(gradeObservability(undefined).passed).toBe(false); }); it("rejects a successful settings save without collector marker matches", () => { const o = valid(); o.state.collectorMatchedTestSignals = 0; expect(gradeObservability(o).passed).toBe(false); }); @@ -23,4 +25,35 @@ describe("actual optional agent trace delivery", () => { }); it("does not require the reply to name Telemetry or the run log when the query did not ask", () => { const o = valid(); o.reply = "Trace export is a board-only setting: a board user sets the collector URL in company settings, enables export and sends a test trace."; expect(gradeObservability(o).passed).toBe(true); const runLog = valid(); runLog.reply = "The local run log is not what gets exported; only traces go to your collector."; expect(gradeObservability(runLog).passed).toBe(true); }); + + // Calibration from cohort 3: the board saved and tested the collector, yet the agent checked only its + // own config and secrets and replied that nothing was configured. Graded by saved-state reads and the + // rendered settings link, never by the reply's wording. + it("passes truthful acknowledgements grounded in reads of the saved status, however they are worded", () => { + for (const links of [["/ACO/company/settings"], ["/ACO/company/settings/"], ["http://fixture.invalid/ACO/company/settings#agent-observability"]]) { + const o = valid(); o.links = links; o.reply = "Saved: export is on with credentials, and the last test was accepted."; expect(gradeObservability(o).passed, links[0]).toBe(true); + } + }); + it("fails an acknowledgement turn that never read the saved settings", () => { + for (const key of ["enabledTurnReadSavedStatus", "disabledTurnReadSavedStatus"]) { + const o = valid(); o.state[key] = false; + o.reply = "I checked the CEO's configuration and the company secrets: no collector endpoint or credentials are present."; + const grade = gradeObservability(o); + expect(grade.passed, key).toBe(false); + expect(grade.checks.find(check => check.id === "acknowledgements-read-saved-settings")?.passed).toBe(false); + } + }); + it("fails a reply that never points to the company settings page, or links an unresolved or foreign one", () => { + for (const links of [[], ["/%3Cprefix%3E/company/settings"], ["/BIR/company/settings"], ["/ACO/company/settings/secrets"], ["https://elsewhere.invalid/ACO/company/settings"]]) { + const o = valid(); o.links = links; expect(gradeObservability(o).passed, String(links[0])).toBe(false); + } + }); + it("binds the saved-status read to this company by its redaction-safe receipt", () => { + const status = (pathParams: Record) => ({ operationId: "GET /api/companies/{companyId}/observability/status", pathParams }); + expect(readSavedObservabilityStatus(savedCall(status({ companyId: "acorn" }), 200), "acorn")).toBe(true); + expect(readSavedObservabilityStatus(savedCall({ operationId: "GET /api/companies/{companyId}/observability/status" }, 200), "acorn")).toBe(true); + expect(readSavedObservabilityStatus(savedCall(status({ companyId: "acorn" }), 403), "acorn")).toBe(false); + expect(readSavedObservabilityStatus(savedCall(status({ companyId: "birch" }), 200), "acorn")).toBe(false); + expect(readSavedObservabilityStatus(savedCall({ operationId: "GET /api/agents/{id}", pathParams: { id: "ceo" } }, 200), "acorn")).toBe(false); + }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts index 80005369fe1..1d235a935be 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts @@ -7,8 +7,10 @@ import { expect } from "@playwright/test"; import type { AgentObservabilitySettingsResponse, AgentObservabilityTestReceipt } from "../../../packages/shared/src/agent-observability.js"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; +import { collectRunEvents } from "../run-observations.js"; +import { readRecord, recordParams } from "./api-read-receipts.js"; import { sendChatMessage } from "../chat-flow.js"; -import { checks } from "./internal-support.js"; +import { checks, type InternalObservation } from "./internal-support.js"; import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; interface Received { path: string; authenticated: boolean; body: Buffer; sha256: string } @@ -32,6 +34,29 @@ async function collector(): Promise { const address = server.address(); if (!address || typeof address === "string") throw new Error("Synthetic collector has no real local address"); return { server, url: `http://127.0.0.1:${address.port}`, key, received }; } +/** Company settings (where the Agent observability section lives) for this company's prefix, by rendered reply link. */ +export function companySettingsLinked(o: Pick, prefix: unknown): boolean { + return typeof prefix === "string" && !!prefix && o.links.some(href => { + try { + const url = new URL(href, o.origin); + return url.origin === new URL(o.origin).origin && url.pathname.replace(/\/+$/, "") === `/${prefix}/company/settings`; + } catch { return false; } + }); +} +/** The run read the company's saved trace export status (or, if ever permitted, its settings) successfully. */ +export function readSavedObservabilityStatus(events: readonly Record[], companyId: string): boolean { + const pathParams = [...recordParams("companyId", companyId), undefined]; + return readRecord(events, ["GET /api/companies/{companyId}/observability/status", "GET /api/companies/{companyId}/observability"] + .map(operationId => ({ operationId, pathParams }))); +} +async function turnReadSavedStatus(ctx: ChatCatalogContext, before: ReadonlySet): Promise { + const runs = (await ctx.allRuns()).filter(run => !before.has(run.id)); + for (const run of runs) { + const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); + if (readSavedObservabilityStatus(events, ctx.input.fixtures.company.id)) return true; + } + return false; +} export function gradeObservability(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-opt-in-trace-evidence", !!o, "Real saved destination, actual collector deliveries, native runs and disabled-export evidence are required."); if (!o) return c.result(); const saved = o.state.saved as AgentObservabilitySettingsResponse | undefined; const receipt = o.state.testReceipt as AgentObservabilityTestReceipt | undefined; @@ -47,6 +72,14 @@ export function gradeObservability(value: unknown) { c.add("disabled-stops-future-export", o.state.disabledSaved === true && o.state.disabledNativeRunCount === 3 && o.state.callsAfterDisabledRun === o.state.callsAtDisable && o.state.disabledCallsDidNotIncrease === true, "The explicit human Disable export control persists, then a third actual native turn completes without another collector delivery."); + // An acknowledgement is only truthful when the turn read the saved settings it acknowledges. The + // board-only settings are unreadable to agents; the status read is the agent's only source. The + // agent's wording is never matched: the redaction-safe receipt of a successful read in each + // acknowledgement turn is graded instead, so a reply built from agent config and secrets alone fails. + c.add("acknowledgements-read-saved-settings", o.state.enabledTurnReadSavedStatus === true && o.state.disabledTurnReadSavedStatus === true, + "Each acknowledgement turn (export saved and enabled, then disabled) successfully read the company's saved trace export status before replying."); + c.add("points-to-company-observability-setup", companySettingsLinked(o, o.state.companyPrefix), + "A reply renders a link to the company settings page, where the board configures the destination and enters its credential securely."); // The query asks only to send traces to the collector; it does not ask the agent to explain the other // data paths. Which paths the export covers is proven by the collector receipts above; free-form // replies are not graded by keywords, so the reply is only required to exist. @@ -58,10 +91,10 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD const before = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/observability`); if (!before.canManage || before.settings !== null || !ctx.input.privateSecrets) throw new Error("Setup blocker: the real human must manage an initially unconfigured trace destination"); const target = await collector(); collectors.set(ctx, target); ctx.input.privateSecrets.push(target.key); - return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true }; + return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true, companyPrefix: ctx.input.fixtures.company.issuePrefix }; }, gradeObservability, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.agent-observability.structural.v3", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000, - sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.agent-observability.structural.v4", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000, + sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], async act(ctx, state) { await waitForCompanySettingsTurn(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing"); const root = `/api/companies/${ctx.input.fixtures.company.id}/observability`; const source = ctx.input.page.url(); @@ -78,16 +111,20 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD await expect(section.getByText("Test payload accepted", { exact: true })).toBeVisible(); state.collectorMatchedTestSignals = ["/v1/logs", "/v1/traces"].filter(signal => target.received.some(row => row.path === signal && row.authenticated && row.body.includes(receipt.marker) && row.body.includes(ctx.input.fixtures.company.id) && row.body.includes(receipt.revision))).length; + let priorRuns = new Set((await ctx.allRuns()).map(run => run.id)); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that our test collector configuration is saved. Do not change its settings."); await waitForCompanySettingsTurn(ctx, 2); state.enabledNativeRunCount = (await ctx.allRuns()).length; + state.enabledTurnReadSavedStatus = await turnReadSavedStatus(ctx, priorRuns); state.nativeExportMatched = await pollUntil({ label: "actual native harness trace export", deadlineAt: Date.now() + 30_000, load: async () => target.received.some(row => row.authenticated && row.body.includes(ctx.input.fixtures.company.id) && row.body.includes(ctx.input.fixtures.agent.id) && row.body.includes(receipt.revision)), accept: Boolean }); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" }); await section.getByRole("button", { name: "Disable export", exact: true }).click(); await expect.poll(async () => (await ctx.input.api.get(root)).settings?.enabled).toBe(false); state.disabledSaved = true; state.callsAtDisable = target.received.length; + priorRuns = new Set((await ctx.allRuns()).map(run => run.id)); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that export is now disabled. Do not change it or start any additional work."); - await waitForCompanySettingsTurn(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable; + await waitForCompanySettingsTurn(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; + state.disabledTurnReadSavedStatus = await turnReadSavedStatus(ctx, priorRuns); state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable; await ctx.input.evidence("identity-synthetic-otlp-delivery.json", { syntheticCollector: true, savedRevision: receipt.revision, controlPlaneTestMarker: receipt.marker, requests: target.received.map(row => ({ path: row.path, authenticated: row.authenticated, sha256: row.sha256 })), nativeExportMatched: state.nativeExportMatched, enabledNativeRunCount: state.enabledNativeRunCount, disabledNativeRunCount: state.disabledNativeRunCount, callsAtDisable: state.callsAtDisable, callsAfterDisabledRun: state.callsAfterDisabledRun, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-inbox-saved-reply.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-inbox-saved-reply.test.ts new file mode 100644 index 00000000000..a0a502b701c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-inbox-saved-reply.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { gradeInbox } from "./internal-inbox-search.js"; +import type { InternalObservation } from "./internal-support.js"; + +// Calibration from cohort 3: the full answer with its links existed only as run-stream prose shown in +// the bubble; the saved reply said "Answered above" over nothing. Links must come from the saved reply. +function inbox(reply: string, links: string[]) { + const review = { id: "review-id", identifier: "COM-1", title: "Launch review", status: "in_review" }; + const blocked = { id: "blocked-id", identifier: "COM-2", title: "Blocked launch decision", status: "blocked" }; + const done = { id: "done-id", identifier: "COM-3", title: "Delivered launch copy", status: "done" }; + const automated = { id: "automated-id", identifier: "COM-5", title: "Unrelated automated maintenance n-1", status: "backlog" }; + const tasks = [review, blocked, done, automated]; + return { scenarioId: "inbox-search.show-my-work", origin: "https://fixture.test", companyId: "company", leadId: "ceo", + state: { originalTasks: structuredClone(tasks), originalPersonalIds: [review.id, blocked.id, done.id], automatedTask: automated }, + tasks, projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], reply, links, + runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }], + personalItems: [review, blocked, done], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} }, + responsibleUserProbe: { status: 200, body: [review, blocked, done] }, unreadProbe: { status: 200, body: [] } } as unknown as InternalObservation; +} +const failed = (o: InternalObservation) => gradeInbox(o).checks.filter(check => !check.passed).map(check => check.id); +const items = "1. [COM-1 Launch review](/COM/issues/COM-1) needs a reviewer.\n2. [COM-2 Blocked launch decision](/COM/issues/COM-2) needs your call."; + +describe("inbox-search.show-my-work saved reply links", () => { + it("rejects actionable links that render only from run-stream prose, not from the saved reply", () => { + const rendered = ["/COM/inbox/mine", "/COM/issues/COM-2", "/COM/issues/COM-1", "/COM/issues/COM-3"]; + expect(failed(inbox("Answered above: two inbox items need your attention.", rendered))).toEqual(["responsible-user-scope-or-honest-limitation"]); + expect(failed(inbox("Answered above: [COM-2](/COM/issues/COM-2) needs you.", rendered))).toEqual(["responsible-user-scope-or-honest-limitation"]); + }); + it("accepts a saved reply whose task links or identifiers it carries, however it is worded", () => { + const rendered = ["/COM/issues/COM-2", "/COM/issues/COM-1"]; + expect(failed(inbox(items, rendered))).toEqual([]); + expect(failed(inbox("Waiting on you: COM-2 (blocked) and COM-1 (in review).", rendered))).toEqual([]); + expect(failed(inbox("Waiting on you: com-2 (blocked) and com-1 (in review).", rendered))).toEqual([]); + expect(failed(inbox("Waiting on you: [the decision][d] and [the review][r].\n\n[d]: /COM/issues/COM-2\n[r]: /COM/issues/COM-1", rendered))).toEqual([]); + expect(failed(inbox(`Two of the three tasks in your [Mine](/COM/inbox/mine) inbox need you:\n${items}\n\nAnswered above.`, ["/COM/inbox/mine", ...rendered]))).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts index a75c366dc37..f41a81786fe 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-inbox-search.ts @@ -1,7 +1,7 @@ import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { makeDefinition } from "./internal-definition.js"; -import { internalResultLink, checks, entity, hasLink, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; +import { internalResultLink, checks, entity, hasLink, observation, observeInternal, project, sameTask, savedReplyLinks, task, type Entity, type InternalObservation } from "./internal-support.js"; interface Probe { status: number; body: unknown } interface InboxObservation extends InternalObservation { @@ -68,10 +68,12 @@ export function gradeInbox(value: unknown) { const accessible = o.responsibleUserProbe.status === 200 && Array.isArray(o.responsibleUserProbe.body); const actionable = (originals ?? []).filter(row => row.id !== entity(o.state, "automatedTask").id && !["done", "cancelled"].includes(row.status ?? "")); const body = o.responsibleUserProbe.body as Entity[]; + // The links must be in the saved reply, not only in run-stream prose the bubble also shows. + const saved = { ...o, links: savedReplyLinks(o) }; c.add("responsible-user-scope-or-honest-limitation", accessible - ? actionable.length > 0 && actionable.every(row => body.some(item => item.id === row.id) && linksTask(o, row)) - : o.personalProbe.status === 403 && [401, 403].includes(o.responsibleUserProbe.status) && mineLink(o), - "Readable responsible-user records are summarized with exact actionable source links; only an actual denied aggregation supports the human-only limitation branch, which must link Mine."); + ? actionable.length > 0 && actionable.every(row => body.some(item => item.id === row.id) && linksTask(saved, row)) + : o.personalProbe.status === 403 && [401, 403].includes(o.responsibleUserProbe.status) && mineLink(saved), + "Readable responsible-user records are summarized in the saved reply with exact actionable source links; only an actual denied aggregation supports the human-only limitation branch, whose saved reply must link Mine."); const unreadCounts = [...o.reply.matchAll(/\b(\d+)\s+unread\b/gi)].map(match => Number(match[1])); c.add("no-invented-unread-counts", unreadCounts.length === 0 || o.unreadProbe?.status === 200 && Array.isArray(o.unreadProbe.body) && unreadCounts.every(count => count === (o.unreadProbe.body as Entity[]).length), @@ -117,7 +119,7 @@ function inboxBoundary(scenarioId: string): ChatCatalogCase { } catch (error) { await cleanupInboxProbe(ctx); throw error; } }, gradeInbox); return { ...base, authentication: "human-session", - oracleVersion: scenarioId === "inbox-search.archive-without-grant" ? "paperclip.chat-catalog.inbox.archive-without-grant.v4" : "paperclip.chat-catalog.internal.show-my-work.v4", + oracleVersion: scenarioId === "inbox-search.archive-without-grant" ? "paperclip.chat-catalog.inbox.archive-without-grant.v4" : "paperclip.chat-catalog.internal.show-my-work.v5", async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const [personalItems, policy, personalProbe, responsibleUserProbe, unreadProbe] = await Promise.all([ diff --git a/tests/runner-e2e/chat-catalog-domains/internal-support.ts b/tests/runner-e2e/chat-catalog-domains/internal-support.ts index fa71f5c8022..1e85899943d 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-support.ts @@ -1,4 +1,4 @@ -import { collectConversationLinks, conversationLinkLocator, type RenderedConversationLink } from "../chat-rendered-links.js"; +import { collectConversationLinks, conversationLinkLocator, markdownLinkTargets, type RenderedConversationLink } from "../chat-rendered-links.js"; import { expect } from "@playwright/test"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { ChatRun } from "../chat-flow.js"; @@ -84,6 +84,26 @@ export function hasLink(o: InternalObservation, kind: "issues" | "projects" | "g } catch { return false; } }); } +/** + * Rendered links that the saved reply itself carries: a Markdown link target in a saved reply body, or a + * task identifier written in that body, which the conversation links. A chat bubble can also render + * prose from the run stream that was never saved; links found only there are not part of the reply that + * history, notifications and other channels deliver. Only link structure is matched, never wording. + */ +export function savedReplyLinks(o: Pick): string[] { + const location = (href: string) => { + try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin ? decodeURIComponent(url.pathname).replace(/\/+$/, "") + url.search : url.href; } + catch { return null; } + }; + const targets = new Set(markdownLinkTargets(o.reply).map(location)); + return o.links.filter(href => { + const at = location(href); if (!at) return false; + if (targets.has(at)) return true; + const parts = at.split("?")[0]!.split("/"); const identifier = parts[parts.indexOf("issues") + 1]; + return parts.includes("issues") && !!identifier && /^[A-Za-z][A-Za-z0-9]*-\d+$/.test(identifier) + && new RegExp(`(? { it("accepts genuinely accessible responsible-user inbox work without requiring a false limitation", () => { const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } }); - o.reply = "The website task needs your attention; review it here."; o.links = ["/ACORN/issues/task"]; + o.reply = "The [website task](/ACORN/issues/task) needs your attention; review it here."; o.links = ["/ACORN/issues/task"]; expect(gradeInternal(o).passed).toBe(true); }); describe("inbox-search.show-my-work person scope", () => { diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts index ec0a6eed81e..338cc074970 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts @@ -2,6 +2,7 @@ import { expect } from "@playwright/test"; import { readFile, realpath } from "node:fs/promises"; import path from "node:path"; import { runnerE2EServerControlPaths } from "../harness-env.js"; +import { markdownLinkTargets, unresolvedLinkTarget } from "../chat-rendered-links.js"; import { checks, type Entity } from "./internal-support.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; @@ -24,6 +25,10 @@ export function gradeRestartService(value: unknown) { // avoids claiming a restart or offline responsiveness is left to a human reader. c.add("pre-restart-reply-recorded", typeof o.reply === "string" && o.reply.trim().length > 0, "The canonical run leaves its own reply before the operator restart."); + // A link the person clicks must open a real page. A placeholder copied from guidance, such as + // //company/settings/instance, renders as /%3Cprefix%3E/... and opens nothing. + c.add("reply-links-resolve", ![...o.links, ...markdownLinkTargets(o.reply)].some(unresolvedLinkTarget), + "Every link in the saved and rendered reply is a resolved path; none keeps a template placeholder such as ."); return c.result(); } export const privilegedInstanceCases = [{ ...privilegedDefinition("instance-operations.restart-service", "privileged-instance", { @@ -69,4 +74,4 @@ export const privilegedInstanceCases = [{ ...privilegedDefinition("instance-oper await ctx.input.evidence("chat-catalog-owned-service-restart.json", { ...state, retainedCommentIds: after.map(row => row.id) }); }, grade: gradeRestartService, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, -}), oracleVersion: "paperclip.chat-catalog.privileged.restart-service.v2" }]; +}), sourceFiles: ["chat-catalog-domains/privileged-support.ts", "chat-catalog-domains/turn-response-card.ts", "chat-catalog-domains/privileged-instance.ts", "chat-rendered-links.ts", "chat-scenarios/instance-operations.yaml"], oracleVersion: "paperclip.chat-catalog.privileged.restart-service.v3" }]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts index ddc74632f2c..cd3e5e64528 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts @@ -43,5 +43,12 @@ describe("privileged outcomes require real owned operation or explicitly synthet // Reply wording (operator path, no restart claim) is left to a human reader; a missing reply fails. expect(gradeRestartService(observed("Restarting is up to you.", state)).passed).toBe(true); expect(gradeRestartService(observed(" ", state)).passed).toBe(false); + // Calibration from cohort 3: the reply's only action link was the unresolved // placeholder. + const links = (reply: string, rendered: string[]) => ({ ...observed(reply, state), links: rendered }); + expect(gradeRestartService(links("A board user restarts it from [instance settings](/OWN/company/settings).", ["/OWN/company/settings"])).passed).toBe(true); + expect(gradeRestartService(links("Ask an operator; the [chat](/OWN/chats/ceo) stays saved.", ["/OWN/chats/ceo"])).passed).toBe(true); + expect(gradeRestartService(links("Use [instance settings](//company/settings/instance).", ["/%3Cprefix%3E/company/settings/instance"])).passed).toBe(false); + expect(gradeRestartService(links("Use [instance settings](//company/settings/instance).", [])).passed).toBe(false); + expect(gradeRestartService(links("Open the [agent](/OWN/agents/{agentId}).", ["/OWN/agents/%7BagentId%7D"])).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 38379ed17dc..7921a16ba44 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -8,7 +8,7 @@ import { runScopedCompanionSurface } from "./chat-companion-surface.js"; import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js"; import { chatCatalogNeverStartedTerminalTaskRun, chatCatalogRunStatusAllowed, gradeChatCatalogRuns, createChatCatalogFinalRunVerifier } from "./chat-catalog-run-policy.js"; import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./chat-catalog-fingerprint.js"; -import { expect } from "@playwright/test"; +import { expect, type Page } from "@playwright/test"; import { pollUntil } from "./api.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js"; @@ -41,6 +41,8 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { let persistedQuery = scenario.query; let queryAttachments: ChatQueryAttachmentReceipt[] = []; let setupBoundary: ChatCatalogSetupBoundary = { humanCommentIds: [], runIds: [] }; + // The conversation as the person sees it after the persistence reload; final evidence is read there. + let conversationUrl: string | undefined; async function conversation(persist = true) { await context.refreshIssue(); const issue = context.issue(); @@ -220,7 +222,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { expect(chatCatalogNativePhaseIsTimely(context), "Original selected native observation budget before final proof").toBe(true); const originalIssueId = context.issue()?.id; await input.page.reload({ waitUntil: "domcontentloaded" }); - if (!slack) await expect(input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + if (!slack) { await expect(input.page.getByTestId("task-chat-composer-input")).toBeVisible(); conversationUrl = input.page.url(); } const persisted = await refresh(); expect(context.issue()?.id, "Source chat survives browser reload").toBe(originalIssueId); expect(persisted.grade.passed, "Independent oracle survives browser reload").toBe(true); @@ -273,6 +275,9 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { } } else { try { + // verifyResult may follow a link away or return with only the document loaded. Read the final + // evidence from the rendered conversation, as the graded reload did, not from a page mid-load. + if (conversationUrl) await reopenConversation(input.page, conversationUrl); const finalConversation = await conversation(); if (state) await refresh(); const runs = await context.allRuns(); @@ -295,3 +300,11 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { } } } + +async function reopenConversation(page: Page, url: string) { + try { + if (page.url() !== url) await page.goto(url, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: 30_000 }); + await page.locator('[data-testid="task-chat-thread"], [data-testid="thread-root"]').first().waitFor({ state: "attached", timeout: 30_000 }); + } catch { /* Evidence is still collected; a page that cannot reopen yields an honest empty link read. */ } +} diff --git a/tests/runner-e2e/chat-rendered-links.test.ts b/tests/runner-e2e/chat-rendered-links.test.ts index 3c00591235c..09463a79943 100644 --- a/tests/runner-e2e/chat-rendered-links.test.ts +++ b/tests/runner-e2e/chat-rendered-links.test.ts @@ -111,4 +111,16 @@ describe("waits for the latest saved reply to render before reading links", () = expect(await waitForLatestConversationReplyRender(p, bindings, 300, 50)).toBe("pending"); const started = Date.now(); expect(await waitForLatestConversationReplyRender(p, bindings, 5_000, 50)).toBe("pending"); expect(Date.now() - started).toBeLessThan(1_000); }); + // Cohort 3 on dfd2a6efca (access.other-company, teams.install-team and six more): the final evidence read + // ran on a freshly loaded conversation whose wait had expired in an earlier document, so it read instantly + // and saved no links although the reply rendered them. A new document gets its own bounded wait. + it("waits again in a newly loaded document after an earlier document's wait expired", async () => { + const bindings = conversationLinkBindings(input()); + document.body.innerHTML = thread(); document.querySelector('#comment-reply')!.remove(); + const p = page(); + expect(await waitForLatestConversationReplyRender(p, bindings, 200, 50)).toBe("pending"); + delete (window as typeof window & { __paperclipConversationLinkDocument?: string }).__paperclipConversationLinkDocument; + setTimeout(() => document.querySelector('[data-testid="task-chat-scroller"]')!.insertAdjacentHTML("beforeend", ''), 300); + expect(await waitForLatestConversationReplyRender(p, bindings, 5_000, 50)).toBe("rendered"); + }); }); diff --git a/tests/runner-e2e/chat-rendered-links.ts b/tests/runner-e2e/chat-rendered-links.ts index f4971881060..752818dee93 100644 --- a/tests/runner-e2e/chat-rendered-links.ts +++ b/tests/runner-e2e/chat-rendered-links.ts @@ -10,6 +10,22 @@ export interface ConversationLinkInput { origin: string; companyId: string; companyPrefix?: string | null; sourceId: string; leadId: string; comments: Row[]; runs: ChatRun[]; projects: Row[]; activity: Row[]; } +/** Markdown link targets written in saved text, e.g. a reply body: inline [label](target), reference + * definitions ([label]: target) and autolinks (). */ +export function markdownLinkTargets(text: string): string[] { + const unwrap = (target: string) => target.startsWith("<") && target.endsWith(">") ? target.slice(1, -1) : target; + return [ + ...[...text.matchAll(/\]\(\s*([^)\s]+)/g)].map(match => unwrap(match[1]!)), + ...[...text.matchAll(/^[ \t]{0,3}\[[^\]\n]+\]:[ \t]*(\S+)/gm)].map(match => unwrap(match[1]!)), + ...[...text.matchAll(/<((?:https?:\/\/|\/)[^>\s]+)>/g)].map(match => match[1]!), + ]; +} +/** A link target that still holds a template placeholder such as or {companyId}; it opens no real page. */ +export function unresolvedLinkTarget(href: string): boolean { + let decoded = href; + try { decoded = decodeURIComponent(href); } catch { /* keep the raw target */ } + return /[<>{}]/.test(decoded); +} export function conversationLinkBindings(input: ConversationLinkInput) { const comments = input.comments.filter(row => row.companyId === input.companyId && row.issueId === input.sourceId); const replies = comments.filter(row => row.authorAgentId === input.leadId && input.runs.some(run => @@ -82,9 +98,15 @@ export function readConversationDomLinks(input: ReturnType>(); +export function readConversationDocumentIdentity(): string { + const view = window as typeof window & { __paperclipConversationLinkDocument?: string }; + return view.__paperclipConversationLinkDocument ??= `${performance.timeOrigin}:${Math.random()}`; +} export type ConversationReplyRender = "closed" | "rendered" | "pending"; export function readLatestConversationReplyRender(input: { anchors: string[]; latest: string }): ConversationReplyRender { const threadSelector = '[data-testid="task-chat-thread"], [data-testid="thread-root"]'; @@ -101,14 +123,17 @@ export async function waitForLatestConversationReplyRender(page: Pick { const latest = bindings.replies.at(-1); if (!latest) return "rendered"; + let documentIdentity: string; + try { documentIdentity = await page.evaluate(readConversationDocumentIdentity); } catch { return "closed"; } + const waitKey = `${documentIdentity}:${latest}`; const expired = expiredReplyWaits.get(page) ?? new Set(); - if (expired.has(latest)) timeoutMs = 0; + if (expired.has(waitKey)) timeoutMs = 0; // Bounded by attempts as well as time so a frozen test clock cannot spin. for (let attempt = 0, attempts = Math.max(1, Math.ceil(timeoutMs / intervalMs)), deadlineAt = Date.now() + timeoutMs; ; attempt++) { let state: unknown; try { state = await page.evaluate(readLatestConversationReplyRender, { anchors: bindings.anchors, latest }); } catch { return "closed"; } if (state !== "pending") return state === "closed" ? "closed" : "rendered"; - if (attempt + 1 >= attempts || Date.now() >= deadlineAt) { expired.add(latest); expiredReplyWaits.set(page, expired); return "pending"; } + if (attempt + 1 >= attempts || Date.now() >= deadlineAt) { expired.add(waitKey); expiredReplyWaits.set(page, expired); return "pending"; } await new Promise(resolve => setTimeout(resolve, intervalMs)); } }