diff --git a/maps/feature-map/documents-artifacts.md b/maps/feature-map/documents-artifacts.md index 7174eed7bdd..a0053b9b158 100644 --- a/maps/feature-map/documents-artifacts.md +++ b/maps/feature-map/documents-artifacts.md @@ -94,6 +94,11 @@ Manual: Edit a paragraph, save, reload, and compare revisions. Add an annotation Automated: [attachment routes](../../server/src/__tests__/issue-attachment-routes.test.ts) and [work products](../../server/src/__tests__/work-products.test.ts) cover data/access behavior; [artifact arrival browser suite](../../tests/e2e/artifact-tab-arrival.spec.ts) targets arrival. [Artifact uploads](../../tests/e2e/artifact-uploads.spec.ts) drives file selection/drop, filename collisions, retry, and preservation of paused/unassigned tasks. +The deployment-targeted [uploaded-input journey](../../tests/runner-e2e/product-journeys/input-file-flow.ts) +adds composer upload, real agent consumption, independent calculation, attributed +saved output and reload checks. Registration and oracle controls do not establish +a live pass on a deployment. + [Native completion feedback](../../server/src/services/native-runtime/native-completion-feedback.test.ts) checks a real saved-document link alongside a verified CSV attachment, including attachment download query and fragment links. It rejects foreign task/company diff --git a/maps/feature-map/inbox-search.md b/maps/feature-map/inbox-search.md index eb11c0d5f85..6ec474f0eb6 100644 --- a/maps/feature-map/inbox-search.md +++ b/maps/feature-map/inbox-search.md @@ -67,6 +67,13 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Seed Automated: [archive routes](../../server/src/__tests__/inbox-archive-routes.test.ts) cover server mutations. [Task-list helpers](../../ui/src/pages/Issues.test.tsx) cover search URLs, pagination, and presentation constants; they do not render the combined Inbox/Tasks page. Reading, archiving, and navigating that page remain manual checks. +The deployment-targeted [personal-triage journey](../../tests/runner-e2e/product-journeys/inbox-triage-flow.ts) +drives Mark as read and Archive, reloads Unread and Mine, independently verifies +unchanged task status, and observes a later Blocked transition resurface the item. +It uses one owned self-assigned task and no agent work; its registration is +separate from a measured Cloud pass. Other actors and company boundaries remain +separate verification cases. + Automated: [Issue service regressions](../../server/src/__tests__/issues-service.test.ts) cover exclusion of historical automatic read cleanup from user participation, unread counts, and activity timestamps while retaining explicit user decisions diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts index de2b46bc3e7..679789a406a 100644 --- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts +++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts @@ -6,6 +6,7 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { HttpError } from "../errors.js"; import { hoistModuleGraph } from "./helpers/hoist-module-graph.js"; +import { inboxBlockedActivity } from "../../../tests/runner-e2e/product-journeys/inbox-triage-fixture.js"; const issueId = "11111111-1111-4111-8111-111111111111"; const companyId = "22222222-2222-4222-8222-222222222222"; @@ -356,6 +357,7 @@ function createRunContextDb( runId: string = ownerRunId, chatBindings: Array<{ id: string; companyId: string; issueId: string; state: string }> = [], emptyWakeQueue = false, + humanInboxFixture?: { userId: string }, ) { const chatBindingQueries: ReturnType[] = []; const runRows = Array.isArray(runAgentOrRows) @@ -413,6 +415,17 @@ function createRunContextDb( transaction: async (callback: (tx: typeof dbStub) => Promise) => callback(dbStub), select: vi.fn((selection: Record = {}) => ({ from: vi.fn((table: Parameters[0]) => { + if (humanInboxFixture && getTableName(table) === "issue_approvals") { + // This disposable human task has no pending approval. A generic + // selected-ID row would invent one and bypass the real route guard. + return { innerJoin: vi.fn(() => ({ where: vi.fn(() => ({ limit: vi.fn(async () => []) })) })) }; + } + if (humanInboxFixture && getTableName(table) === "company_memberships") { + return { where: vi.fn((condition: SQL) => ({ limit: vi.fn(async () => + new PgDialect().sqlToQuery(condition).params.includes(humanInboxFixture.userId) + ? [{ id: "inbox-member" }] : []), + })) }; + } if (getTableName(table) === "case_handoffs") return emptyCaseHandoffQuery(); if (emptyWakeQueue && getTableName(table) === "agent_wakeup_requests") { return buildQuery({}, false, true); @@ -1833,6 +1846,32 @@ describe("agent issue mutation checkout ownership", () => { ); }); + it.each(["bare blocked update", "human-owned inbox fixture"] as const)( + "applies the production blocked contract to the %s", async kind => { + const actor = boardActor(); + const existing = makeIssue({ status: "in_progress", assigneeAgentId: null, assigneeUserId: actor.userId }); + mockIssueService.getById.mockResolvedValue(existing); + mockIssueService.update.mockImplementation(async (_id: string, patch: Record) => ({ ...existing, ...patch })); + const db = createRunContextDb({}, ownerAgentId, ownerRunId, [], false, { userId: actor.userId }); + const payload = kind === "bare blocked update" ? { status: "blocked" } : inboxBlockedActivity(actor.userId); + + const res = await request(await createApp(actor, db)).patch(`/api/issues/${issueId}`).send(payload); + + if (kind === "bare blocked update") { + expect(res.status, JSON.stringify(res.body)).toBe(422); + expect(res.body.error).toBe("Entering blocked requires unresolved blockers, a pending interaction/approval, or unblockDescriptor"); + expect(mockIssueService.update).not.toHaveBeenCalled(); + } else { + expect(res.status, JSON.stringify(res.body)).toBe(200); + expect(res.body).toMatchObject({ status: "blocked", unblockDescriptor: { + owner: { userId: actor.userId }, action: "Review the QA inbox task before continuing.", + } }); + expect(mockIssueService.update).toHaveBeenCalledWith(issueId, expect.objectContaining(payload)); + } + expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled(); + }, + ); + it("rejects peer-agent status updates that would clear a recovery action they do not own", async () => { mockIssueService.getById.mockResolvedValue( makeIssue({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" }), diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index ed8c2d6e55b..f4db3bc0341 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -23,6 +23,8 @@ never start a server or depend on a tenant filesystem. | Journey | Required starting state | Observed outcome | | --- | --- | --- | +| `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | +| `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 173610b4afc..1bee4c69c0e 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,13 @@ # Runner E2E security for a public repository +The deployment-targeted `file-input-answer` journey uploads a small synthetic +CSV through the normal task composer. Its reference and numeric inputs are +generated per attempt and occur only in the file. The independent fixture and +run/revision receipts remain in the private Cloud bundle; the public QA report +exports only the existing closed checks and authenticated task screenshot. +The `inbox-triage` journey creates and changes only its owned self-assigned task. +It uses no model, no foreign user records, and no direct database writes. + The explicit `remote-workspace-storage` cell snapshots only its synthetic projectless task folder in an owned disposable sandbox. Its prompt creates a small Python project, local Git history without a remote, and an untracked random @@ -625,3 +633,5 @@ S3 prefix, and generated secrets between controllers. It removes the entire old home after graceful handoff. Its read-only observer does not create recovery authority. Provider credentials still enter through encrypted fixture secrets. Local rolling smoke receipts explicitly exclude active-agent and hosted proof. + +The personal inbox journey requires a separately authenticated, registered QA human author. Its target provider exposes one controlled comment operation on the exact campaign-owned task. The public oracle independently reads the saved comment and verifies its distinct human author, exact content, company, task, and absence of an agent run. The private provider closes the isolated author context during evidence-first cleanup. No provider or native run is allowed for this browser-only journey; agent cross-task execution guards remain intact. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index d6e41627457..49b77c23da0 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -126,7 +126,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 8e93e71f946..2fb117aa68f 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -18,7 +18,9 @@ function associate(suites: readonly string[], cases: Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 13d423d5740..6a0e3a19c57 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,8 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "inbox-triage", label: "Read, archive and resurface an owned inbox task", requires: ["company", "board-user", "task-write", "inbox-author"], capabilities: ["inbox-search/views", "inbox-search/triage"], paid: false }, + { id: "file-input-answer", label: "Upload a reference file and verify the agent uses its contents", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments", "documents-artifacts/documents"], paid: true }, { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, { id: "project-task-context", label: "Create a project and reopen its self-assigned task", requires: ["company", "board-user", "task-write"], capabilities: ["projects/lifecycle", "projects/task-context", "tasks/create"], paid: false }, { id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index f3c20ccb6f7..8ddc2911323 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -221,7 +221,7 @@ describe("deployment journey contracts", () => { onboarding.productJourney!.operations.modelCostUsd=null; expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull(); }); - it.each(["task-properties", "task-create", "project-task-context"])("requires real pass evidence and forbids model work in browser-only %s passes", caseId => { + it.each(["inbox-triage", "task-properties", "task-create", "project-task-context"])("requires real pass evidence and forbids model work in browser-only %s passes", caseId => { const browser = result(); browser.caseId = caseId; browser.executionId = `product-journeys.local-qa.local.${caseId}`; diff --git a/tests/runner-e2e/product-journeys/fixtures.ts b/tests/runner-e2e/product-journeys/fixtures.ts index 2d057b3cf69..b26ff61ddbb 100644 --- a/tests/runner-e2e/product-journeys/fixtures.ts +++ b/tests/runner-e2e/product-journeys/fixtures.ts @@ -11,7 +11,7 @@ export function journeyFixtures(session: TargetSession, campaignId?: string) { id: "ready-context", async setup() { return ready ??= await (preparation ??= session.prepare()); }, }); - for (const id of ["company", "board-user", "task-write", "agent", "skill-write", "review-service"]) registry.register({ + for (const id of ["company", "board-user", "task-write", "agent", "skill-write", "review-service", "inbox-author"]) registry.register({ id, dependencies: ["ready-context"], async setup(resolved) { const context = resolved.get("ready-context") as JourneyContext; diff --git a/tests/runner-e2e/product-journeys/inbox-author-contract.test.ts b/tests/runner-e2e/product-journeys/inbox-author-contract.test.ts new file mode 100644 index 00000000000..e3786c6c205 --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-author-contract.test.ts @@ -0,0 +1,21 @@ +import { expect, it } from "vitest"; +import { inboxAuthorCommentMatches } from "./inbox-author-contract.js"; + +const expected = { companyId: "company", issueId: "issue", ownerUserId: "owner", authorUserId: "fixture-human", body: "Fresh fixture update" }; +const comment = { id: "comment", companyId: "company", issueId: "issue", body: expected.body, + authorUserId: "fixture-human", authorAgentId: null, createdByRunId: null }; +it("requires the independently authenticated human's exact saved activity", () => { + expect(inboxAuthorCommentMatches(comment, expected)).toBe(true); +}); +it.each([ + undefined, null, { ...comment, id: "" }, { ...comment, companyId: "foreign" }, { ...comment, issueId: "foreign" }, + { ...comment, body: "stale or replaced activity" }, { ...comment, authorUserId: "owner" }, + { ...comment, authorUserId: null, authorAgentId: "fixture-agent" }, + { ...comment, authorAgentId: "agent" }, { ...comment, createdByRunId: "run" }, +])("rejects wrong or missing external human activity %#", value => { + expect(inboxAuthorCommentMatches(value, expected)).toBe(false); +}); +it("rejects a fixture claiming the current user's identity or no author", () => { + expect(inboxAuthorCommentMatches({ ...comment, authorUserId: "owner" }, { ...expected, authorUserId: "owner" })).toBe(false); + expect(inboxAuthorCommentMatches(comment, { ...expected, authorUserId: "" })).toBe(false); +}); diff --git a/tests/runner-e2e/product-journeys/inbox-author-contract.ts b/tests/runner-e2e/product-journeys/inbox-author-contract.ts new file mode 100644 index 00000000000..482b87969b3 --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-author-contract.ts @@ -0,0 +1,24 @@ +/** A separately authenticated human fixture. Credentials remain with its target provider. */ +export interface InboxAuthorFixture { + userId: string; + postComment(input: { issueId: string; body: string }): Promise; +} +export interface InboxAuthorComment { + id: string; + companyId: string; + issueId: string; + body: string; + authorUserId: string | null; + authorAgentId: string | null; + createdByRunId?: string | null; +} + +/** Neither an owner-authored comment nor claimed alternate attribution proves unread activity. */ +export function inboxAuthorCommentMatches(value: InboxAuthorComment | null | undefined, expected: { + companyId: string; issueId: string; ownerUserId: string; authorUserId: string; body: string; +}) { + return Boolean(value && typeof value.id === "string" && value.id + && expected.authorUserId && expected.authorUserId !== expected.ownerUserId && expected.body + && value.companyId === expected.companyId && value.issueId === expected.issueId && value.body === expected.body + && value.authorUserId === expected.authorUserId && value.authorAgentId === null && value.createdByRunId == null); +} diff --git a/tests/runner-e2e/product-journeys/inbox-triage-evidence.test.ts b/tests/runner-e2e/product-journeys/inbox-triage-evidence.test.ts new file mode 100644 index 00000000000..b186eaaf577 --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-triage-evidence.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { inboxArchivePersisted, inboxTaskMatches, inboxTaskResurfaced } from "./inbox-triage-evidence.js"; + +describe("personal inbox triage evidence", () => { + const expected = { id: "owned-task", companyId: "company", userId: "reader", status: "in_progress" }; + const task = { id: expected.id, companyId: expected.companyId, assigneeUserId: expected.userId, + assigneeAgentId: null, status: expected.status, isUnreadForMe: false }; + const archived = { ...task, archivedAt: "2020-01-01T12:00:00.000Z" }; + it("requires persisted personal archive state and actual exclusion from Mine", () => { + expect(inboxArchivePersisted(archived, expected, [])).toBe(true); + expect(inboxArchivePersisted(task, expected, [])).toBe(false); + expect(inboxArchivePersisted(archived, expected, [task])).toBe(false); + expect(inboxArchivePersisted({ ...archived, archivedAt: "not-a-time" }, expected, [])).toBe(false); + }); + it.each([{ id: "foreign-task" }, { companyId: "foreign-company" }, { assigneeUserId: "other-user" }, + { assigneeAgentId: "unexpected-agent" }, { status: "done" }])("rejects changed ownership or completion %j", change => { + expect(inboxTaskMatches({ ...task, ...change }, expected)).toBe(false); + expect(inboxArchivePersisted({ ...archived, ...change }, expected, [])).toBe(false); + }); + it("requires one correctly owned visible item after a relevant new status transition", () => { + const blocked = { ...task, status: "blocked" }, updated = { ...expected, status: "blocked" }; + expect(inboxTaskResurfaced(blocked, updated, [blocked])).toBe(true); + expect(inboxTaskResurfaced(blocked, updated, [])).toBe(false); + expect(inboxTaskResurfaced(blocked, updated, [blocked, blocked])).toBe(false); + expect(inboxTaskResurfaced({ ...blocked, archivedAt: archived.archivedAt }, updated, [blocked])).toBe(false); + expect(inboxTaskResurfaced(blocked, updated, [{ ...blocked, assigneeUserId: "other-user" }])).toBe(false); + expect(inboxTaskMatches(undefined, expected)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/inbox-triage-evidence.ts b/tests/runner-e2e/product-journeys/inbox-triage-evidence.ts new file mode 100644 index 00000000000..71638edfc6c --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-triage-evidence.ts @@ -0,0 +1,23 @@ +export interface InboxTaskEvidence { + id: string; companyId: string; assigneeUserId: string | null; assigneeAgentId: string | null; + status: string; isUnreadForMe?: boolean; archivedAt?: string | null; +} + +export function inboxTaskMatches(task: InboxTaskEvidence | undefined, + expected: { id: string; companyId: string; userId: string; status: string }): boolean { + return Boolean(task && task.id === expected.id && task.companyId === expected.companyId + && task.assigneeUserId === expected.userId && task.assigneeAgentId === null && task.status === expected.status); +} + +export function inboxArchivePersisted(task: InboxTaskEvidence | undefined, + expected: { id: string; companyId: string; userId: string; status: string }, visible: InboxTaskEvidence[]): boolean { + return inboxTaskMatches(task, expected) && typeof task!.archivedAt === "string" + && Number.isFinite(Date.parse(task!.archivedAt)) && !visible.some(row => row.id === expected.id); +} + +export function inboxTaskResurfaced(task: InboxTaskEvidence | undefined, + expected: { id: string; companyId: string; userId: string; status: string }, visible: InboxTaskEvidence[]): boolean { + const matches = visible.filter(row => row.id === expected.id); + return inboxTaskMatches(task, expected) && task!.archivedAt == null && matches.length === 1 + && inboxTaskMatches(matches[0], expected) && matches[0]!.archivedAt == null; +} diff --git a/tests/runner-e2e/product-journeys/inbox-triage-fixture.ts b/tests/runner-e2e/product-journeys/inbox-triage-fixture.ts new file mode 100644 index 00000000000..553d67f7f10 --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-triage-fixture.ts @@ -0,0 +1,10 @@ +/** Ordinary user-owned relevant activity for an archived inbox task. */ +export function inboxBlockedActivity(userId: string) { + return { + status: "blocked" as const, + unblockDescriptor: { + owner: { userId }, + action: "Review the QA inbox task before continuing.", + }, + }; +} diff --git a/tests/runner-e2e/product-journeys/inbox-triage-flow.test.ts b/tests/runner-e2e/product-journeys/inbox-triage-flow.test.ts new file mode 100644 index 00000000000..dba6cbb1d4a --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-triage-flow.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from "vitest"; +import { inboxStep, openPersonalInbox } from "./inbox-triage-flow.js"; +import { JourneyError, type JourneyContext } from "./types.js"; + +type Probe = { visible?: () => boolean; count?: () => number; value?: () => string }; +vi.mock("@playwright/test", () => ({ expect: (probe: Probe) => ({ + toBeVisible: async () => { if (!probe.visible?.()) throw new Error("private browser diagnostic"); }, + toHaveCount: async (expected: number) => { if (probe.count?.() !== expected) throw new Error("private browser diagnostic"); }, + toHaveValue: async (expected: string) => { if (probe.value?.() !== expected) throw new Error("private browser diagnostic"); }, +}) })); + +function fixture(options: { loading?: boolean; listVisible?: boolean; ignoreClear?: boolean } = {}) { + let query = "saved global search"; + const fills: string[] = [], visits: string[] = []; + const search = { + visible: () => true, value: () => query, + fill: async (value: string) => { fills.push(value); if (!options.ignoreClear) query = value; }, + }; + const list = { visible: () => options.listVisible !== false }; + const context = { + api: { origin: "https://qa.example.test" }, company: { issuePrefix: "QA" }, + page: { + goto: async (url: string) => { visits.push(url); }, + getByPlaceholder: () => search, + getByRole: () => ({ count: () => options.loading ? 1 : 0 }), + getByText: () => ({ visible: () => false }), + locator: () => ({ or: () => ({ first: () => list }) }), + }, + } as unknown as JourneyContext; + // Production search results can lack personal archive controls. A normal + // view is required even when an ID selector could locate a search result. + return { context, fills, visits, personalControlsAvailable: () => query === "" }; +} + +describe("personal inbox navigation", () => { + it.each(["mine", "unread"] as const)("clears global search before acting in the %s view", async view => { + const f = fixture(); + expect(f.personalControlsAvailable()).toBe(false); + await openPersonalInbox(f.context, view); + expect(f.visits).toEqual([`https://qa.example.test/QA/inbox/${view}`]); + expect(f.fills).toEqual([""]); + expect(f.personalControlsAvailable()).toBe(true); + }); + + it.each([{ loading: true }, { listVisible: false }, { ignoreClear: true }])("does not prove absence from an unready or searched view: %j", async options => { + await expect(openPersonalInbox(fixture(options).context, "unread")).rejects.toMatchObject({ + outcome: "failed", code: "inbox_unread_view_unavailable", + }); + }); +}); + +describe("inbox operation diagnostics", () => { + it("returns the original successful observation", async () => { + const observation = { isUnreadForMe: true }; + expect(await inboxStep("inbox_unread_projection_failed", async () => observation)).toBe(observation); + }); + + it("retains a closed code without raw browser values", async () => { + const failure = await inboxStep("inbox_unread_projection_failed", async () => { + throw new Error("https://private.example/callback?token=DO_NOT_RETAIN"); + }).catch(error => error); + expect(failure).toBeInstanceOf(JourneyError); + expect(failure.code).toBe("inbox_unread_projection_failed"); + expect(JSON.stringify(failure)).not.toContain("DO_NOT_RETAIN"); + }); + + it("preserves an independently detected missing task projection", async () => { + const missing = new JourneyError("failed", "inbox_owned_task_projection_missing"); + await expect(inboxStep("inbox_unread_projection_failed", async () => { throw missing; })).rejects.toBe(missing); + }); +}); diff --git a/tests/runner-e2e/product-journeys/inbox-triage-flow.ts b/tests/runner-e2e/product-journeys/inbox-triage-flow.ts new file mode 100644 index 00000000000..baca8ab7ef8 --- /dev/null +++ b/tests/runner-e2e/product-journeys/inbox-triage-flow.ts @@ -0,0 +1,108 @@ +import { expect } from "@playwright/test"; +import { inboxArchivePersisted, inboxTaskMatches, inboxTaskResurfaced, type InboxTaskEvidence } from "./inbox-triage-evidence.js"; +import { inboxAuthorCommentMatches, type InboxAuthorComment } from "./inbox-author-contract.js"; +import { inboxBlockedActivity } from "./inbox-triage-fixture.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +export async function inboxStep(code: string, operation: () => Promise): Promise { + try { return await operation(); } + catch (error) { + if (error instanceof JourneyError) throw error; + // Browser errors may carry form values and URLs. Retain only the fixed + // operation code, including which independent read failed. + throw new JourneyError("failed", code); + } +} + +export async function openPersonalInbox(context: Pick, view: "mine" | "unread") { + const { page, api, company } = context; + await inboxStep(`inbox_${view}_view_unavailable`, async () => { + await page.goto(`${api.origin}/${company.issuePrefix}/inbox/${view}`, { waitUntil: "domcontentloaded" }); + const search = page.getByPlaceholder(/^Search (?:inbox|tasks)(?:…|\.\.\.)$/i); + await expect(search).toBeVisible(); + // Global search supplements the inbox with other results whose read and + // archive controls differ. Act on the real personal view, scoped by row ID. + await search.fill(""); + await expect(search).toHaveValue(""); + // An absent row is meaningful only after this actual view has loaded its + // list or the correct empty state; a skeleton cannot prove exclusion. + await expect(page.getByRole("status", { name: "Loading tasks", exact: true })).toHaveCount(0); + const empty = page.getByText(view === "mine" ? "Inbox zero." : "No new inbox items.", { exact: true }); + await expect(page.locator("#main-content [data-inbox-item]").or(empty).first()).toBeVisible(); + }); +} + +export async function inboxTriage(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence) { + const { api, page, company, user } = context; + const author = context.fixtures?.inboxAuthor; + if (!author?.userId || author.userId === user.id) throw new JourneyError("unsupported", "inbox_human_author_required"); + const title = `Journey QA inbox triage ${campaignId}`; + await session.own({ kind: "issue", name: title, journeyId: "inbox-triage" }); + const issue = await inboxStep("inbox_task_creation_failed", () => api.post(`/api/companies/${company.id}/issues`, { + title, description: "Disposable personal inbox triage task.", assigneeUserId: user.id, assigneeAgentId: null, status: "in_progress", + })); + await session.own({ kind: "issue", name: title, id: issue.id, journeyId: "inbox-triage" }); + const route = `/api/issues/${issue.id}`, expected = { id: issue.id, companyId: company.id, userId: user.id, status: "in_progress" }; + const check = (id: string, passed: boolean) => { + evidence.checks.push({ id, passed }); + if (!passed) throw new JourneyError("failed", id.replaceAll("-", "_")); + }; + // Personal unread fields are available on the user-aware list projection. + const read = async () => { + const rows = await api.get(`/api/companies/${company.id}/issues?assigneeUserId=me&touchedByUserId=me&q=${encodeURIComponent(title)}`); + const own = rows.find(row => row.id === issue.id); + if (!own) throw new JourneyError("failed", "inbox_owned_task_projection_missing"); + return own; + }; + const mine = () => api.get(`/api/companies/${company.id}/issues?touchedByUserId=me&inboxArchivedByUserId=me`); + const row = page.locator(`[data-issue-row-details="${issue.id}"]`).first(); + // Agent writes correctly require a real heartbeat run. The model-free inbox + // fixture uses a second normal human session rather than inventing a run. + const body = `Inbox fixture update ${campaignId}`; + const posted = await inboxStep("inbox_external_comment_failed", () => author.postComment({ issueId: issue.id, body })); + const saved = await inboxStep("inbox_external_comment_read_failed", () => api.get(`${route}/comments/${posted.id}`)); + const expectedAuthor = { companyId: company.id, issueId: issue.id, ownerUserId: user.id, authorUserId: author.userId, body }; + check("inbox-external-human-activity", posted.id === saved.id + && inboxAuthorCommentMatches(posted, expectedAuthor) && inboxAuthorCommentMatches(saved, expectedAuthor)); + await inboxStep("inbox_mark_unread_failed", () => api.delete(`${route}/read`)); + await inboxStep("inbox_unread_projection_failed", () => expect.poll(async () => (await read()).isUnreadForMe).toBe(true)); + await openPersonalInbox(context, "unread"); + await inboxStep("inbox_unread_row_missing", () => expect(row).toBeVisible()); + await inboxStep("inbox_mark_read_failed", () => row.getByRole("button", { name: "Mark as read", exact: true }).click()); + await inboxStep("inbox_read_projection_failed", () => expect.poll(async () => (await read()).isUnreadForMe).toBe(false)); + await inboxStep("inbox_unread_reload_failed", () => page.reload()); + await openPersonalInbox(context, "unread"); + await inboxStep("inbox_read_row_still_visible", () => expect(row).toHaveCount(0)); + check("inbox-read-persisted", inboxTaskMatches(await read(), expected) && (await read()).isUnreadForMe === false); + + await openPersonalInbox(context, "mine"); + await inboxStep("inbox_mine_row_missing", () => expect(row).toBeVisible()); + await inboxStep("inbox_archive_control_failed", async () => { + await row.hover(); + await row.getByRole("button", { name: "Archive", exact: true }).click(); + }); + await inboxStep("inbox_archive_projection_failed", () => expect.poll(async () => inboxArchivePersisted(await read(), expected, await mine())).toBe(true)); + await inboxStep("inbox_mine_reload_failed", () => page.reload()); + await openPersonalInbox(context, "mine"); + await inboxStep("inbox_archived_row_still_visible", () => expect(row).toHaveCount(0)); + check("inbox-archive-persisted", inboxArchivePersisted(await read(), expected, await mine())); + check("inbox-archive-preserved-task-status", (await read()).status === "in_progress"); + + // A real new Blocked transition is relevant activity. The fixture supplies the + // event, while the production archive policy must independently resurface it. + await inboxStep("inbox_relevant_activity_failed", () => api.patch(route, inboxBlockedActivity(user.id))); + const blocked = { ...expected, status: "blocked" }; + await inboxStep("inbox_resurface_projection_failed", () => expect.poll(async () => inboxTaskResurfaced(await read(), blocked, await mine())).toBe(true)); + await openPersonalInbox(context, "mine"); + await inboxStep("inbox_resurfaced_row_missing", () => expect(row).toBeVisible()); + await inboxStep("inbox_resurfaced_reload_failed", () => page.reload()); + await openPersonalInbox(context, "mine"); + await inboxStep("inbox_resurfaced_row_lost", () => expect(row).toBeVisible()); + check("inbox-relevant-activity-resurfaced", inboxTaskResurfaced(await read(), blocked, await mine())); + check("no-agent-work", (await api.get(`${route}/runs`)).length === 0); + // Leave the owned task open for the launcher's authenticated task screenshot. + await inboxStep("inbox_task_reopen_failed", async () => { + await page.goto(`${api.origin}/${company.issuePrefix}/issues/${issue.identifier}`, { waitUntil: "domcontentloaded" }); + await expect(page.getByRole("heading", { name: title, exact: true })).toBeVisible(); + }); +} diff --git a/tests/runner-e2e/product-journeys/input-file-evidence.test.ts b/tests/runner-e2e/product-journeys/input-file-evidence.test.ts new file mode 100644 index 00000000000..3cba18503ae --- /dev/null +++ b/tests/runner-e2e/product-journeys/input-file-evidence.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { inputFileAttachmentMatches, inputFileFixture, inputFileSummaryMatches } from "./input-file-evidence.js"; + +describe("uploaded input evidence", () => { + const fixture = inputFileFixture("fresh-unpredictable-reference"); + it("grades the uploaded values rather than a fixed sample answer", () => { + expect(inputFileSummaryMatches(JSON.stringify(fixture.expected), fixture.expected)).toBe(true); + expect(inputFileSummaryMatches(`\`\`\`json\n${JSON.stringify(fixture.expected)}\n\`\`\``, fixture.expected)).toBe(true); + const other = inputFileFixture("a-different-input"); + expect(inputFileSummaryMatches(JSON.stringify(other.expected), fixture.expected)).toBe(false); + }); + it.each([ + null, "", "I read the file successfully", "{}", "[]", + JSON.stringify({ ...fixture.expected, acceptedRows: 5 }), + JSON.stringify({ ...fixture.expected, acceptedUnits: fixture.expected.acceptedUnits + 1 }), + JSON.stringify({ ...fixture.expected, reference: "another-task" }), + JSON.stringify({ ...fixture.expected, unsupported: "extra assertion" }), + ])("rejects missing, wrong or unsupported output %j", body => { + expect(inputFileSummaryMatches(body, fixture.expected)).toBe(false); + }); + it("requires the exact uploaded bytes on the owned task and preserves their identity", () => { + const attachment = { id: "input-file", companyId: "company", issueId: "task", + originalFilename: fixture.filename, originatingRunId: null, byteSize: fixture.bytes.length }; + const input = { companyId: "company", issueId: "task", fixture, downloaded: fixture.bytes }; + expect(inputFileAttachmentMatches(attachment, input)).toBe(true); + expect(inputFileAttachmentMatches(undefined, input)).toBe(false); + for (const change of [{ companyId: "foreign" }, { issueId: "foreign" }, { originatingRunId: "agent-created-replacement" }, + { originalFilename: "different.csv" }, { byteSize: fixture.bytes.length + 1 }]) { + expect(inputFileAttachmentMatches({ ...attachment, ...change }, input)).toBe(false); + } + expect(inputFileAttachmentMatches(attachment, { ...input, downloaded: Buffer.from("replacement") })).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/input-file-evidence.ts b/tests/runner-e2e/product-journeys/input-file-evidence.ts new file mode 100644 index 00000000000..fb670d6ffbc --- /dev/null +++ b/tests/runner-e2e/product-journeys/input-file-evidence.ts @@ -0,0 +1,48 @@ +import { createHash } from "node:crypto"; + +export interface InputFileFixture { + filename: string; + bytes: Buffer; + expected: { reference: string; acceptedRows: number; acceptedUnits: number }; +} + +/** Answers exist only in the uploaded bytes, never in the task prompt. */ +export function inputFileFixture(reference: string): InputFileFixture { + const seed = createHash("sha256").update(reference).digest(); + const rows = Array.from({ length: 5 }, (_, index) => ({ + item: `item-${index + 1}`, units: seed[index]! + 1, + decision: index === 1 || index === 4 ? "declined" : "accepted", + })); + return { + filename: "order-review.csv", + bytes: Buffer.from(`reference,item,units,decision\n${rows.map(row => `${reference},${row.item},${row.units},${row.decision}`).join("\n")}\n`), + expected: { reference, acceptedRows: 3, + acceptedUnits: rows.filter(row => row.decision === "accepted").reduce((sum, row) => sum + row.units, 0) }, + }; +} + +export function inputFileSummaryMatches(body: unknown, expected: InputFileFixture["expected"]): boolean { + if (typeof body !== "string") return false; + const text = body.trim(); + const json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value: unknown = JSON.parse(json); + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const row = value as Record; + return Object.keys(row).sort().join(",") === "acceptedRows,acceptedUnits,reference" + && row.reference === expected.reference && row.acceptedRows === expected.acceptedRows + && row.acceptedUnits === expected.acceptedUnits; + } catch { return false; } +} + +export interface InputFileAttachment { + id: string; companyId: string; issueId: string; originalFilename: string; + originatingRunId: string | null; byteSize: number; +} + +export function inputFileAttachmentMatches(attachment: InputFileAttachment | undefined, + input: { companyId: string; issueId: string; fixture: InputFileFixture; downloaded: Buffer }): boolean { + return Boolean(attachment?.id && attachment.companyId === input.companyId && attachment.issueId === input.issueId + && attachment.originalFilename === input.fixture.filename && attachment.originatingRunId === null + && attachment.byteSize === input.fixture.bytes.length && input.downloaded.equals(input.fixture.bytes)); +} diff --git a/tests/runner-e2e/product-journeys/input-file-flow.ts b/tests/runner-e2e/product-journeys/input-file-flow.ts new file mode 100644 index 00000000000..691f7a0891a --- /dev/null +++ b/tests/runner-e2e/product-journeys/input-file-flow.ts @@ -0,0 +1,85 @@ +import { randomBytes, createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { inputFileAttachmentMatches, inputFileFixture, inputFileSummaryMatches, type InputFileAttachment } from "./input-file-evidence.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "file-input-answer"; + +export async function runInputFile(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const fixture = inputFileFixture(randomBytes(16).toString("hex")); + const file = path.join(options.directory, fixture.filename); + await writeFile(file, fixture.bytes, { mode: 0o600 }); + const key = "upload-summary", title = `Journey QA uploaded input ${campaignId}`; + const prompt = `Read the attached ${fixture.filename}. Count only rows whose decision is accepted and sum their units. ` + + `Save a task document with key ${key} containing JSON with exactly these fields: reference (the reference in the file), ` + + "acceptedRows (the accepted row count), and acceptedUnits (their units total). Do not count declined rows. " + + "Keep the original attachment unchanged, then finish the task. No other tasks or agents are needed."; + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt, workMode: "standard", + requireExplicitTitle: true, attachments: [file], + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + const inputs = await waitFor(options.deadlineAt, () => context.api.get(`${route}/attachments`), + rows => rows.some(row => row.originalFilename === fixture.filename), "uploaded_input_missing"); + const original = inputs.find(row => row.originalFilename === fixture.filename); + const inputBytes = original ? await context.api.bytes(`/api/attachments/${original.id}/content`) : Buffer.alloc(0); + check("input-upload-bytes-preserved", inputs.filter(row => row.originalFilename === fixture.filename).length === 1 && inputFileAttachmentMatches(original, + { companyId: context.company.id, issueId: created.issueId, fixture, downloaded: inputBytes })); + + const state = await waitFor(options.deadlineAt, async () => { + const [currentIssue, runs, comments, interactions] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), + context.api.get(`${route}/comments?order=asc`), context.api.get>(`${route}/interactions`), + ]); + if (runs.length > 1) throw new JourneyError("failed", "input_file_run_limit_exceeded"); + const taskRuns = await Promise.all(runs.map(row => context.api.get(`/api/heartbeat-runs/${row.runId}`))); + const value = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + validateAgentTaskState(context, created.issueId, value, 1); + return value; + }, value => value.currentIssue.status === "done" && value.taskRuns.length === 1 + && value.taskRuns[0]!.status === "succeeded" && value.interactions.every(row => row.status !== "pending"), + "input_file_task_not_completed"); + const document = await context.api.get(`${route}/documents/${key}`); + const revision = await context.api.get(`${route}/documents/${key}/revisions/${document.latestRevisionId}`); + check("input-content-used-correctly", inputFileSummaryMatches(document.body, fixture.expected)); + check("input-result-attributed-to-run", document.companyId === context.company.id && document.issueId === created.issueId + && document.createdByAgentId === agent.id && document.createdByUserId === null + && revision.documentId === document.id && revision.createdByRunId === state.taskRuns[0]!.id + && revision.createdByAgentId === agent.id && inputFileSummaryMatches(revision.body, fixture.expected)); + const url = `${context.api.origin}/${context.company.issuePrefix}/issues/${state.currentIssue.identifier ?? created.issueId}`; + await context.page.goto(url, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + const saved = context.page.locator(`#document-${key}`); + await expect(saved).toContainText(fixture.expected.reference); + const reopened = await context.api.get(`${route}/documents/${key}`); + const retainedInputs = await context.api.get(`${route}/attachments`); + const retained = retainedInputs.find(row => row.id === original?.id); + check("input-result-reopened", reopened.latestRevisionId === document.latestRevisionId + && inputFileSummaryMatches(reopened.body, fixture.expected)); + check("original-input-retained", retainedInputs.filter(row => row.originalFilename === fixture.filename).length === 1 && inputFileAttachmentMatches(retained, + { companyId: context.company.id, issueId: created.issueId, fixture, + downloaded: retained ? await context.api.bytes(`/api/attachments/${retained.id}/content`) : Buffer.alloc(0) })); + check("configured-model-and-environment", true); // Every observed run was independently checked above. + await writeFile(path.join(options.directory, "input-file-checks.json"), JSON.stringify({ + inputSha256: createHash("sha256").update(fixture.bytes).digest("hex"), expected: fixture.expected, + issueId: created.issueId, attachmentId: original!.id, documentId: document.id, + revisionId: revision.id, runId: state.taskRuns[0]!.id, + }, null, 2), { mode: 0o600 }); +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index a6cc1027c36..6a0226f1d42 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -17,13 +17,15 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runInputFile } from "./input-file-flow.js"; +import { inboxTriage } from "./inbox-triage-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; import { resolveRunnerE2ESource } from "../source.js"; import { validateRetainedRunnerResult } from "../result-validation.js"; import { validateJourneyCollection } from "./collection.js"; -import { captureProductScreenshot } from "./product-screenshot.js"; +import { captureProductScreenshot, isProductScreenshotUrl } from "./product-screenshot.js"; import { validateOnboardingArtifact } from "./artifact-evidence.js"; const cleanupGraceMs = 600_000; @@ -108,7 +110,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -204,7 +206,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + if (id === "file-input-answer") await runInputFile(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); else if (id === "followup-while-running") await runActiveFollowup(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, @@ -221,7 +225,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await mkdir(directory, { recursive: true, mode: 0o700 }); await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); - } else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); + } else if (id === "inbox-triage") await inboxTriage(ready, session!, campaignId, evidence); + else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); else if (id === "task-document-persistence") await taskDocumentPersistence(ready, session!, campaignId, evidence); else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); else if (id === "task-search-filter") await taskSearchFilter(ready, session!, campaignId, evidence); @@ -247,8 +252,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri // Retain failed product screens only after tenant setup, never login screens. if (ready && phase === "journey") { try { - const url = new URL(ready.page.url()); - if (url.origin === ready.api.origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname)) { + if (isProductScreenshotUrl(ready.page.url(), ready.api.origin)) { const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}-failed.png`), deadlineAt); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 37f079e6a4f..486b6c19f0e 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -4,8 +4,8 @@ import { captureProductScreenshot } from "./product-screenshot.js"; const origin = "https://qa.example.test"; afterEach(() => vi.restoreAllMocks()); -function fixture(options: { mounted?: boolean; redirect?: string; afterMount?(): void } = {}) { - let mounted = false, url = `${origin}/QA/issues/QA-1`; +function fixture(options: { mounted?: boolean; redirect?: string; initialPath?: string; afterMount?(): void } = {}) { + let mounted = false, url = `${origin}${options.initialPath ?? "/QA/issues/QA-1"}`; const screenshot = vi.fn(async (_input: { path: string; timeout: number }) => {}); const waitFor = vi.fn(async (_input: { state: string; timeout: number }) => { mounted = options.mounted !== false; if (options.redirect) url = options.redirect; options.afterMount?.(); }); const locator = vi.fn(() => ({ waitFor, isVisible: async () => mounted, screenshot })); @@ -13,6 +13,17 @@ function fixture(options: { mounted?: boolean; redirect?: string; afterMount?(): return { page, locator, waitFor, screenshot }; } describe("product screenshot boundary", () => { + it.each(["/QA/inbox/unread", "/QA/routines"])("retains an authenticated journey failure on %s", async initialPath => { + const f = fixture({ initialPath }); + await captureProductScreenshot(f.page, origin, "failed.png"); + expect(f.screenshot).toHaveBeenCalledTimes(1); + }); + it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export"])("excludes unrelated or authentication paths: %s", async initialPath => { + const f = fixture({ initialPath }); + await expect(captureProductScreenshot(f.page, origin, "failed.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.waitFor).not.toHaveBeenCalled(); + expect(f.screenshot).not.toHaveBeenCalled(); + }); it("waits for Layout to mount after the document navigation completes", async () => { const f = fixture(); await captureProductScreenshot(f.page, origin, "shot.png"); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index f8387e447e2..cedfe14a17a 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -1,11 +1,13 @@ import type { Page } from "@playwright/test"; import { JourneyError } from "./types.js"; +export function isProductScreenshotUrl(value: string, origin: string): boolean { + const url = new URL(value); + return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines)(\/|$)/.test(url.pathname); +} + export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { - const permitted = () => { - const url = new URL(page.url()); - return url.origin === origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname); - }; + const permitted = () => isProductScreenshotUrl(page.url(), origin); const timeout = (maximum: number) => { const remaining = deadlineAt - Date.now(); if (remaining <= 0) throw new JourneyError("failed", "product_screen_unavailable"); diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 1ce269fac61..1da065336c6 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,5 +1,5 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ -export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, +export const agentJourneyRuns = { "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { @@ -9,6 +9,8 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "inbox-triage": ["inbox-external-human-activity", "inbox-read-persisted", "inbox-archive-persisted", "inbox-archive-preserved-task-status", "inbox-relevant-activity-resurfaced", "no-agent-work"], + "file-input-answer": ["input-upload-bytes-preserved", "input-content-used-correctly", "input-result-attributed-to-run", "input-result-reopened", "original-input-retained", "configured-model-and-environment"], "task-properties": ["assignment-persisted", "no-agent-work"], "task-create": ["browser-created-task-persisted", "no-agent-work"], "task-document-persistence": ["browser-document-created", "document-revision-history-preserved", "document-reopened-after-reload", "no-agent-work"], diff --git a/tests/runner-e2e/product-journeys/task-document-flow.ts b/tests/runner-e2e/product-journeys/task-document-flow.ts index 8e6327f68c2..32dd89d69c5 100644 --- a/tests/runner-e2e/product-journeys/task-document-flow.ts +++ b/tests/runner-e2e/product-journeys/task-document-flow.ts @@ -2,7 +2,7 @@ import { expect } from "@playwright/test"; import { taskDocumentHistoryPasses, taskDocumentMatches, type TaskDocumentExpectation, type TaskDocumentRevision, type TaskDocumentSnapshot } from "./task-document-evidence.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; -async function openDocumentAuthoring({ page }: JourneyContext) { +export async function openDocumentAuthoring({ page }: JourneyContext) { const create = page.getByRole("button", { name: "New document", exact: true }); const properties = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).first(); const artifacts = page.getByRole("tab", { name: /^Artifacts/ }).first(); diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index df7289fb732..bb3b9bc38c0 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; @@ -84,6 +84,7 @@ export interface JourneyContext { agent?: { id: string; name: string; provider: string; model: string; runtimeMode: "native" | "legacy" }; fixtures?: { backlogTask?: PreparedBacklogTask; + inboxAuthor?: import("./inbox-author-contract.js").InboxAuthorFixture; reviewService?: { connectionId: string; name: string; marker: string; invocationCount(): number }; }; } diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 73e726d6491..e709c7439ba 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -434,7 +434,7 @@ export function validateRetainedRunnerResult( if (!evidence) return invalid("result.productJourney"); const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); - if (["task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (["inbox-triage", "task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt");