From a4308b9791855f99732cec796785f5ea8fe4e294 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:07:21 -0500 Subject: [PATCH] test(e2e): add browser project and task context journey (#18353) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - People group tasks into projects and use those projects to keep work in context. > - Deployment journeys already test task creation and assignment. > - They do not yet prove that a new project and its task keep the same saved relationship after reload. > - This pull request adds an explicit browser journey for that gap. > - Independent API reads and visible task properties must agree on the exact records and selected settings. > - The journey uses no model runs. Each deployment must pass before a daily plan can include it. ## Linked Issues or Issue Description **What existing behavior does this improve?** The deployment-targeted Product E2E suite can now check project creation and task context through the production dialogs. **Current behavior** The suite covers standalone task creation and assignment. It does not prove that a newly created project contains the task selected in the composer, or that the relation survives reload. **Proposed behavior** Select `project-task-context`. Create a unique project and a self-assigned task with that project selected. Reopen the project, open its task, and reload. Require exact saved company, project, title, prompt, user assignment and standard work mode. Require no agent runs before or after reload. **Reason and benefit** The same journey can qualify different deployments. It catches lost project links and incorrect saved context without a model call or a separate QA-only test implementation. ## What Changed - Add the browser flow and an independent saved-state oracle with positive and negative cases. - Scope project and task navigation to unique links in the main content so sidebar links cannot bypass list coverage. - Bind reopened routes to saved project and task references. Verify the main project heading and visible task content instead of a hidden duplicate sidebar label. Keep browser failures as closed stage codes. - Register the journey, its closed success checks, and its source files in the definition hash. Default model-matrix selections stay unchanged. - Associate qualification-report coverage with project lifecycle, project task context and task creation. Reject retained browser-only passes containing run IDs, provider/model metadata or native/legacy execution. - Add `project` to the existing durable ownership resource contract. A target provider remains responsible for cleanup of its own project fixtures. - Update discovery expectations, the deployment journey guide and the projects feature map. ## Verification - Focused runner tests: 110 passed across the new oracle, discovery, journey contracts and existing report subprocess checks. The first sandboxed attempt blocked the existing report subprocess IPC socket; the same selection passed with local IPC enabled. - Late review regression checks: both report association and unexpected browser-only model work reproduced as failing tests before the fix. The final contract, saved-state oracle and report catalog selection passed all 72 tests. Runner typecheck passed again after these changes. - Browser readback: the old project-name selector failed in Chromium when a hidden sidebar label preceded the production project heading. All six browser regression cases now pass, including missing rows, wrong project/task destinations and a wrong project property. The fixture uses the production heading and task-row overlay conventions. Credential-free Node 24 imports pass. - Current-master integration: 208 focused catalog, project/search, campaign and reporting tests passed. All nine project/search browser regressions passed. Runner typecheck, map checks and Node 24 imports for both journeys passed. The combined catalog, dispatch, source hashes, success checks, browser-only guard and feature associations retain both selections. - The first deployment qualification created both owned records but failed during later readback with a generic browser code. Authentication, setup, collection and cleanup passed. That failed grade is retained. The corrected source subsequently passed a fresh attached QA campaign; the original failure remains preserved. - New oracle alone: 7 tests passed after the final catalog mapping change. - `node scripts/check-maps.mjs` — passed; structural checks only. - Credential-free `--suite product-journeys --list` — passed and lists this browser journey. - `node node_modules/typescript/bin/tsc -p tests/runner-e2e/tsconfig.json` — passed after building required plugin, runner and eval-kernel declarations. The initial fresh-checkout check failed because those declarations were not built. - All current-head CI gates passed, including repository typecheck, tests, build and browser checks. Greptile reviewed the exact head at 5/5 with no unresolved findings. Fresh attached QA passed at `f19557c9eebde14c2def37195d6b154c23eb9690`: [sanitized report](https://pages.paperclip.ing/evals-qa-smoke-41119722363-1/). All five independent checks and authentication, setup, journey, collection and cleanup passed; zero agent runs started. ## Risks - This adds an opt-in test and changes no production behavior or schema. UI control changes can fail the journey. - The target provider must support durable `project` ownership before it can run this case. Project cleanup must settle owned tasks, reject foreign linked tasks and archive the project while preserving history. - Source registration alone does not include the candidate in any saved daily plan. A fresh pass against that deployment is required first. ## Model Used - OpenAI Codex, GPT-6 family, with code inspection and tool execution. The host does not expose the exact underlying model revision or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above (no duplicate journey found) - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- maps/feature-map/projects.md | 8 +++ tests/runner-e2e/PRODUCT-JOURNEYS.md | 13 ++++ tests/runner-e2e/catalog.test.ts | 2 +- tests/runner-e2e/journey-features.ts | 1 + tests/runner-e2e/playwright-support.config.ts | 2 +- tests/runner-e2e/product-journeys/catalog.ts | 1 + .../product-journeys/contracts.test.ts | 16 +++-- tests/runner-e2e/product-journeys/launch.ts | 4 +- .../project-context-evidence.test.ts | 61 +++++++++++++++++++ .../project-context-evidence.ts | 51 ++++++++++++++++ .../product-journeys/project-context-flow.ts | 53 ++++++++++++++++ .../project-context-readback.ts | 48 +++++++++++++++ .../product-journeys/success-contract.ts | 1 + tests/runner-e2e/product-journeys/types.ts | 4 +- .../project-context-readback.spec.ts | 51 ++++++++++++++++ tests/runner-e2e/report-catalog.test.ts | 10 ++- tests/runner-e2e/result-validation.ts | 2 +- 17 files changed, 316 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/project-context-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/project-context-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/project-context-flow.ts create mode 100644 tests/runner-e2e/product-journeys/project-context-readback.ts create mode 100644 tests/runner-e2e/project-context-readback.spec.ts diff --git a/maps/feature-map/projects.md b/maps/feature-map/projects.md index b793d57d678..382c3992682 100644 --- a/maps/feature-map/projects.md +++ b/maps/feature-map/projects.md @@ -44,6 +44,14 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use Automated: [projects UI](../../ui/src/pages/Projects.test.tsx) and [project detail](../../ui/src/pages/ProjectDetail.test.tsx) are component tests. +Automated: the explicit [`project-task-context` deployment journey](../../tests/runner-e2e/product-journeys/project-context-flow.ts) +creates a project and a self-assigned task through the production dialogs. It +reopens both, verifies the project task list and task properties after reload, +and independently checks the saved company, project, prompt, assignment and +standard work mode with zero agent runs. [Oracle tests](../../tests/runner-e2e/product-journeys/project-context-evidence.test.ts) +reject lost links and mismatched records. This describes coverage; each target +still needs its own live pass before daily promotion. + Manual: Create a project, add a task from its view, and reload. Verify the task belongs to that project, list counts are plausible, and links preserve company scope. Archive only the disposable project and inspect list/task behavior. ### `repository-settings` diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index aadfb7a7e7b..59bd750a262 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -24,6 +24,7 @@ never start a server or depend on a tenant filesystem. | Journey | Required starting state | Observed outcome | | --- | --- | --- | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | +| `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | | `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact | @@ -38,6 +39,18 @@ The shared Studio helper waits for the original saved file body before typing. An editor control can mount before the file query completes; editing that empty draft can otherwise lose the original content when the file arrives. +The project/task journey records durable project and task ownership before each +creation and immediately saves the response IDs. Its project must retain its +planned state. Independent project-filtered task reads must contain the exact +new task. The final task view must show the same project after reload. Browser +readback uses the main project heading and visible task content rather +than a duplicate sidebar label. It binds navigation to the saved project URL key +and task identifier. Closed failure codes identify project-list, project-open, +task-open and property-readback failures without retaining Playwright page text. A Cloud +provider must clean only these owned resources, settle owned tasks before +archiving the project, and preserve their history. Catalog registration does not +admit this candidate to a saved daily plan; a real deployment pass is required. + The search/filter journey creates only campaign-owned, board-assigned tasks through the public API. The target and completed control share a unique search marker; a second Backlog control has a different marker. Search and status selection use diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index eba2916d746..2dc4e45836c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 630e59d1c05..683f170c1e7 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record ({ expect: Object.assign( @@ -220,12 +221,17 @@ describe("deployment journey contracts", () => { onboarding.productJourney!.operations.modelCostUsd=null; expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull(); }); - it("requires real pass evidence and forbids model work in browser-only passes", () => { - expect(()=>validateRetainedRunnerResult(result())).not.toThrow(); - const missing = result(); missing.productJourney!.checks=[]; + it.each(["task-properties", "task-create", "project-task-context"])("requires real pass evidence and forbids model work in browser-only %s passes", caseId => { + const browser = result(); + browser.caseId = caseId; + browser.executionId = `product-journeys.local-qa.local.${caseId}`; + browser.productJourney!.checks = journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })); + expect(()=>validateRetainedRunnerResult(browser)).not.toThrow(); + const missing = structuredClone(browser); missing.productJourney!.checks=[]; expect(()=>validateRetainedRunnerResult(missing)).toThrow(); - const paid = result(); paid.runIds=["unexpected"]; - expect(()=>validateRetainedRunnerResult(paid)).toThrow(); + const forbidden: Array> = [{ runIds: ["unexpected"] }, { provider: "openai" }, { model: "unexpected" }, + { runtimeMode: "native" }, { runtimeMode: "legacy" }]; + for (const work of forbidden) expect(()=>validateRetainedRunnerResult({ ...browser, ...work })).toThrow("result.browserOnly"); }); it("keeps browser-only reporting free of invented model calls and rejects public publication", () => { const value=result(); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 0ee1e314d98..aa65ce9b560 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -11,6 +11,7 @@ import { journeyFixtures } from "./fixtures.js"; import { taskProperties } from "./task-properties.js"; import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; +import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; @@ -103,7 +104,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "agent-flows.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -208,6 +209,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file)); } }); } else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); + else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); else if (id === "task-search-filter") await taskSearchFilter(ready, session!, campaignId, evidence); else await taskProperties(ready, session!, campaignId, evidence); } diff --git a/tests/runner-e2e/product-journeys/project-context-evidence.test.ts b/tests/runner-e2e/product-journeys/project-context-evidence.test.ts new file mode 100644 index 00000000000..1caa6986eb2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-evidence.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, preparationPreview, productJourneys } from "./catalog.js"; +import { projectContextEvidencePasses, type ProjectContextObservation } from "./project-context-evidence.js"; +import { journeyRequiredChecks } from "./success-contract.js"; + +function observation(): ProjectContextObservation { + const project = { id: "project", companyId: "company", name: "Journey project campaign", status: "planned", archivedAt: null }; + const task = { id: "task", companyId: "company", title: "Journey task campaign", description: "Review this myself.", projectId: "project", + assigneeUserId: "user", assigneeAgentId: null, workMode: "standard" }; + return { companyId: "company", userId: "user", projectId: "project", projectName: project.name, + taskId: "task", taskTitle: task.title, prompt: task.description, + projectBeforeReload: { ...project }, projectAfterReload: { ...project }, taskBeforeReload: { ...task }, taskAfterReload: { ...task }, + projectTasks: [{ ...task }], runsBeforeReload: [], runsAfterReload: [] }; +} + +describe("project/task context independent saved-state oracle", () => { + it("accepts exact persisted project, task, assignment and work mode with no work runs", () => { + expect(projectContextEvidencePasses(observation())).toBe(true); + }); + it.each(["projectBeforeReload", "projectAfterReload"] as const)("rejects a wrong or missing %s", key => { + for (const patch of [{ id: "foreign" }, { companyId: "foreign" }, { name: "another project" }, { status: "completed" }, { archivedAt: "2099-01-01T00:00:00Z" }]) { + const saved = observation(); + Object.assign(saved[key], patch); + expect(projectContextEvidencePasses(saved)).toBe(false); + } + const missing = observation(); + Reflect.deleteProperty(missing, key); + expect(projectContextEvidencePasses(missing)).toBe(false); + }); + it.each(["taskBeforeReload", "taskAfterReload"] as const)("rejects plausible false passes in %s", key => { + for (const patch of [{ id: "foreign" }, { companyId: "foreign" }, { title: "wrong" }, { description: "lost prompt" }, { projectId: null }, + { projectId: "foreign" }, { assigneeUserId: "foreign" }, { assigneeAgentId: "agent" }, { workMode: "ask" }]) { + const saved = observation(); + Object.assign(saved[key], patch); + expect(projectContextEvidencePasses(saved)).toBe(false); + } + const missing = observation(); + Reflect.deleteProperty(missing, key); + expect(projectContextEvidencePasses(missing)).toBe(false); + }); + it("requires exact project-filtered membership and rejects any native work before or after reload", () => { + const changes: Array<(saved: ProjectContextObservation) => void> = [ + saved => { saved.projectTasks = []; }, saved => { saved.projectTasks.push({ ...saved.projectTasks[0]! }); }, + saved => { saved.projectTasks[0]!.projectId = "foreign"; }, saved => { saved.projectTasks[0]!.companyId = "foreign"; }, + saved => { saved.projectTasks[0]!.id = "another task"; }, saved => { saved.runsBeforeReload = [{ id: "run" }]; }, + saved => { saved.runsAfterReload = [{ id: "run" }]; }, saved => { Reflect.deleteProperty(saved, "runsAfterReload"); }, + saved => { Reflect.deleteProperty(saved, "projectTasks"); }, saved => { saved.projectId = ""; }, + ]; + for (const change of changes) { const saved = observation(); change(saved); expect(projectContextEvidencePasses(saved)).toBe(false); } + }); + it("is discoverable as a browser journey with zero model budget or native work", () => { + const config = parseTargetConfig({ version: 1, id: "project-qa", mode: "managed-self-hosted", journeys: ["project-task-context"], + executionEnvironment: "local", baseline: { features: {}, connections: [] }, limits: { durationMs: 300_000, modelBudgetCents: 0, maxRuns: 0 } }); + const entry = productJourneys.find(journey => journey.id === "project-task-context")!; + expect(entry.paid).toBe(false); + expect(entry.requires).toEqual(["company", "board-user", "task-write"]); + expect(preparationPreview(config).journeys).toEqual([entry]); + expect(journeyRequiredChecks[entry.id]).toEqual(["browser-created-project-persisted", "browser-created-project-task-persisted", + "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/project-context-evidence.ts b/tests/runner-e2e/product-journeys/project-context-evidence.ts new file mode 100644 index 00000000000..393b9e82053 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-evidence.ts @@ -0,0 +1,51 @@ +interface ProjectSnapshot { + id: string; + companyId: string; + name: string; + status: string; + archivedAt: string | null; + urlKey?: string | null; +} +interface TaskSnapshot { + id: string; + companyId: string; + title: string; + description: string | null; + projectId: string | null; + assigneeUserId: string | null; + assigneeAgentId: string | null; + workMode: string; + identifier?: string | null; +} +export interface ProjectContextObservation { + companyId: string; + userId: string; + projectId: string; + projectName: string; + taskId: string; + taskTitle: string; + prompt: string; + projectBeforeReload: ProjectSnapshot; + projectAfterReload: ProjectSnapshot; + taskBeforeReload: TaskSnapshot; + taskAfterReload: TaskSnapshot; + projectTasks: TaskSnapshot[]; + runsBeforeReload: unknown[]; + runsAfterReload: unknown[]; +} + +/** Compare independent saved reads against the submitted UI values, not the POST echo. */ +export function projectContextEvidencePasses(observed: ProjectContextObservation): boolean { + const { companyId, userId, projectId, projectName, taskId, taskTitle, prompt } = observed; + if (![companyId, userId, projectId, projectName, taskId, taskTitle, prompt].every(value => typeof value === "string" && value.length)) return false; + const projectMatches = (project: ProjectSnapshot) => project?.id === projectId && project.companyId === companyId + && project.name === projectName && project.status === "planned" && project.archivedAt === null; + const taskMatches = (task: TaskSnapshot) => task?.id === taskId && task.companyId === companyId && task.title === taskTitle + && task.description === prompt && task.projectId === projectId && task.assigneeUserId === userId + && task.assigneeAgentId === null && task.workMode === "standard"; + return projectMatches(observed.projectBeforeReload) && projectMatches(observed.projectAfterReload) + && taskMatches(observed.taskBeforeReload) && taskMatches(observed.taskAfterReload) + && Array.isArray(observed.projectTasks) && observed.projectTasks.length === 1 && taskMatches(observed.projectTasks[0]!) + && Array.isArray(observed.runsBeforeReload) && observed.runsBeforeReload.length === 0 + && Array.isArray(observed.runsAfterReload) && observed.runsAfterReload.length === 0; +} diff --git a/tests/runner-e2e/product-journeys/project-context-flow.ts b/tests/runner-e2e/product-journeys/project-context-flow.ts new file mode 100644 index 00000000000..2dae9213854 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-flow.ts @@ -0,0 +1,53 @@ +import { expect } from "@playwright/test"; +import { createTaskThroughUi } from "../user-actions.js"; +import { projectContextEvidencePasses, type ProjectContextObservation } from "./project-context-evidence.js"; +import { openProjectTaskThroughList, reopenProjectThroughList } from "./project-context-readback.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +export async function projectTaskContext(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence) { + const { page, api, company } = context; + const projectName = `Journey QA project ${campaignId}`; + const taskTitle = `Journey QA project task ${campaignId}`; + const prompt = `Keep this task assigned to me in ${projectName}. I will review it myself.`; + const journeyId = "project-task-context" as const; + await page.goto(`${api.origin}/${company.issuePrefix}/projects`); + await page.getByRole("button", { name: "Add Project", exact: true }).first().click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("textbox", { name: "Project name", exact: true }).fill(projectName); + await session.own({ kind: "project", name: projectName, journeyId }); + const responsePromise = page.waitForResponse(response => response.request().method() === "POST" + && new URL(response.url()).pathname === `/api/companies/${company.id}/projects`); + await dialog.getByRole("button", { name: "Create project", exact: true }).click(); + const response = await responsePromise; + if (response.status() !== 201) throw new JourneyError("failed", "project_ui_creation_failed"); + const createdProject = await response.json(); + if (typeof createdProject.id !== "string" || !createdProject.id || createdProject.companyId !== company.id || createdProject.name !== projectName) { + throw new JourneyError("failed", "project_ui_creation_mismatch"); + } + await session.own({ kind: "project", name: projectName, id: createdProject.id, journeyId }); + await expect(dialog).not.toBeVisible(); + const projectRoute = `/api/projects/${createdProject.id}`; + const projectBeforeReload = await api.get(projectRoute); + const task = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: taskTitle, prompt, workMode: "standard", projectName, requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: taskTitle, journeyId }) }); + await session.own({ kind: "issue", name: taskTitle, id: task.issueId, journeyId }); + const taskRoute = `/api/issues/${task.issueId}`; + const taskBeforeReload = await api.get(taskRoute); + const runsBeforeReload = await api.get(`${taskRoute}/runs`); + + // Open the project from its list, then open its task from that project's Tasks view. + await reopenProjectThroughList({ page, origin: api.origin, issuePrefix: company.issuePrefix, projectName, + projectRef: projectBeforeReload.urlKey ?? createdProject.id }); + const projectTasks = await api.get(`/api/companies/${company.id}/issues?projectId=${createdProject.id}&limit=100&offset=0`); + await openProjectTaskThroughList({ page, origin: api.origin, issuePrefix: company.issuePrefix, projectName, taskTitle, prompt, + taskRef: taskBeforeReload.identifier ?? task.issueId }); + const observed: ProjectContextObservation = { + companyId: company.id, userId: context.user.id, projectId: createdProject.id, projectName, + taskId: task.issueId, taskTitle, prompt, projectBeforeReload, taskBeforeReload, projectTasks, runsBeforeReload, + projectAfterReload: await api.get(projectRoute), taskAfterReload: await api.get(taskRoute), runsAfterReload: await api.get(`${taskRoute}/runs`), + }; + if (!projectContextEvidencePasses(observed)) throw new JourneyError("failed", "project_task_context_mismatch"); + evidence.checks.push(...["browser-created-project-persisted", "browser-created-project-task-persisted", + "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"].map(id => ({ id, passed: true }))); +} diff --git a/tests/runner-e2e/product-journeys/project-context-readback.ts b/tests/runner-e2e/product-journeys/project-context-readback.ts new file mode 100644 index 00000000000..498e9c7a5f3 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-readback.ts @@ -0,0 +1,48 @@ +import { expect, type Page } from "@playwright/test"; +import { JourneyError } from "./types.js"; + +type BrowserStage = "project_list" | "project_open" | "project_task_open" | "project_property"; +async function browserStep(stage: BrowserStage, action: () => Promise): Promise { + try { return await action(); } + catch (error) { + if (error instanceof JourneyError) throw error; + // Playwright errors can contain page content. Retain only the closed stage. + throw new JourneyError("failed", `project_context_${stage}_failed`); + } +} + +export async function reopenProjectThroughList(input: { page: Page; origin: string; issuePrefix: string; projectName: string; projectRef: string }) { + const { page, origin, issuePrefix, projectName, projectRef } = input; + await browserStep("project_list", () => page.goto(`${origin}/${issuePrefix}/projects`)); + await browserStep("project_open", async () => { + const link = page.locator("main#main-content").getByRole("link").filter({ hasText: projectName }); + await expect(link).toHaveCount(1); + await link.click(); + const expected = `${origin}/${issuePrefix}/projects/${encodeURIComponent(projectRef)}/issues`; + await expect.poll(() => { const url = new URL(page.url()); return `${url.origin}${url.pathname}`; }).toBe(expected); + await page.reload(); + await expect(page.locator("main#main-content").getByRole("heading", { name: projectName, exact: true })).toBeVisible(); + }); +} + +export async function openProjectTaskThroughList(input: { page: Page; origin: string; issuePrefix: string; projectName: string; taskTitle: string; taskRef: string; prompt: string }) { + const { page, origin, issuePrefix, projectName, taskTitle, taskRef, prompt } = input; + await browserStep("project_task_open", async () => { + const link = page.locator("main#main-content").getByRole("link").filter({ hasText: taskTitle }); + await expect(link).toHaveCount(1); + await link.click(); + const expected = `${origin}/${issuePrefix}/issues/${encodeURIComponent(taskRef)}`; + await expect.poll(() => { const url = new URL(page.url()); return `${url.origin}${url.pathname}`; }).toBe(expected); + await page.reload(); + const main = page.locator("main#main-content"); + await expect(main.getByText(taskTitle, { exact: true }).filter({ visible: true }).first()).toBeVisible(); + await expect(main.getByText(prompt, { exact: true }).filter({ visible: true }).first()).toBeVisible(); + }); + await browserStep("project_property", async () => { + const properties = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }).first(); + if (await properties.isVisible()) await properties.click(); + const projectProperty = page.locator("[data-property-row]").filter({ has: page.locator('[data-property-label="Project"]') }).filter({ visible: true }); + await expect(projectProperty).toHaveCount(1); + await expect(projectProperty.locator("[data-property-value]")).toContainText(projectName); + }); +} diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index aadf872905a..8e68da0a091 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -4,6 +4,7 @@ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "b export const journeyRequiredChecks: Record = { "task-properties": ["assignment-persisted", "no-agent-work"], "task-create": ["browser-created-task-persisted", "no-agent-work"], + "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], "plan-revise-accept": ["plan-revision-and-approval", "plan-final-answer", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index b932f301391..ec7746cf303 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -2,7 +2,7 @@ import type { BrowserContext, Page } from "@playwright/test"; import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; -export type JourneyId = "task-properties" | "task-create" | "ask-question" | "cloud-onboarding" +export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; @@ -112,7 +112,7 @@ export interface TargetSession { /** Publish the observed finalization failure without releasing resources or leases. */ checkpointEvidence?(input?: TargetLifecycleInput): Promise; /** Durable intent before creation; receipt immediately after creation. */ - own(resource: { kind: "issue" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId }): Promise; + own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId }): Promise; } export interface TargetProvider { version: 1; diff --git a/tests/runner-e2e/project-context-readback.spec.ts b/tests/runner-e2e/project-context-readback.spec.ts new file mode 100644 index 00000000000..fb04df62bea --- /dev/null +++ b/tests/runner-e2e/project-context-readback.spec.ts @@ -0,0 +1,51 @@ +import { expect, test, type Page } from "@playwright/test"; +import { openProjectTaskThroughList, reopenProjectThroughList } from "./product-journeys/project-context-readback.js"; + +const origin = "https://project-context.test", projectName = "Journey QA project owned-campaign"; +const taskTitle = "Journey QA project task owned-campaign", prompt = "I will review this task myself."; +const projectPath = "/QA/projects/owned-project-f962db91/issues", taskPath = "/QA/issues/QA-201"; +const projectInput = { origin, issuePrefix: "QA", projectName, projectRef: "owned-project-f962db91" }; +const taskInput = { origin, issuePrefix: "QA", projectName, taskTitle, prompt, taskRef: "QA-201" }; + +async function fixture(page: Page, changes: { missingProject?: boolean; wrongProject?: boolean; missingTask?: boolean; wrongTask?: boolean; wrongProperty?: boolean } = {}) { + const requests: string[] = []; + await page.route(`${origin}/**`, async route => { + const path = new URL(route.request().url()).pathname; + requests.push(path); + // Layout can retain a hidden sidebar label. ProjectDetail renders its + // production InlineEditor as h2; IssueRow has an overlay link with sr-only text. + const sidebar = ``; + const body = path === "/QA/projects" ? `${sidebar}
${changes.missingProject ? "" : `${projectName}`}
` + : path.includes("/projects/") ? `${sidebar}

${projectName}

${changes.missingTask ? "" : ``}
` + : `${sidebar}

${taskTitle}

${prompt}

`; + await route.fulfill({ contentType: "text/html", body: `${body}` }); + }); + return requests; +} + +test("reopens and reloads the owned project and task despite hidden duplicate sidebar labels", async ({ page }) => { + const requests = await fixture(page); + await reopenProjectThroughList({ ...projectInput, page }); + await openProjectTaskThroughList({ ...taskInput, page }); + expect(requests.filter(path => path === projectPath)).toHaveLength(2); + expect(requests.filter(path => path === taskPath)).toHaveLength(2); + await expect(page.locator("main#main-content [data-property-value]")).toContainText(projectName); +}); + +for (const [name, changes, code] of [ + ["missing project list row", { missingProject: true }, "project_context_project_open_failed"], + ["wrong project route with a matching label", { wrongProject: true }, "project_context_project_open_failed"], +] as const) test(`rejects ${name} with a closed stage code`, async ({ page }) => { + await fixture(page, changes); + await expect(reopenProjectThroughList({ ...projectInput, page })).rejects.toMatchObject({ outcome: "failed", code, message: code }); +}); + +for (const [name, changes, code] of [ + ["missing task in the project", { missingTask: true }, "project_context_project_task_open_failed"], + ["wrong task route with a matching label", { wrongTask: true }, "project_context_project_task_open_failed"], + ["wrong saved project property", { wrongProperty: true }, "project_context_project_property_failed"], +] as const) test(`rejects ${name} with a closed stage code`, async ({ page }) => { + await fixture(page, changes); + await page.goto(`${origin}${projectPath}`); + await expect(openProjectTaskThroughList({ ...taskInput, page })).rejects.toMatchObject({ outcome: "failed", code, message: code }); +}); diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 2560c28971a..28929783eb1 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -10,7 +10,7 @@ import { discoverReportCatalog } from "./report-catalog.js"; import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; -import { journeyCapabilities } from "./journey-features.js"; +import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -91,6 +91,14 @@ describe("trusted report catalog discovery", () => { expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]); }); + it("counts the project/task deployment journey toward projects and task creation coverage", () => { + const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } }; + expect(journeyCapabilities(project).map(capability => capability.id)).toEqual(["projects/lifecycle", "projects/task-context", "tasks/create"]); + expect(journeyFeatureIds(project)).toEqual(["projects", "tasks"]); + expect(journeyCapabilities({ ...project, suite: { ...project.suite, id: "unrecognized-suite" } })).toEqual([]); + }); + it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => { const summary = campaign(); expect(summary.suites).toEqual([ diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index af9d9bdfc1e..8399905bf7e 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -434,7 +434,7 @@ export function validateRetainedRunnerResult( if (!evidence) return invalid("result.productJourney"); const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); - if (["task-properties", "task-create", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (["task-properties", "task-create", "project-task-context", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (checked.runIds?.length !== agentJourneyRuns[checked.caseId as keyof typeof agentJourneyRuns] || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt");