diff --git a/.github/scripts/tests/feature-map.test.mjs b/.github/scripts/tests/feature-map.test.mjs new file mode 100644 index 00000000000..3194903f701 --- /dev/null +++ b/.github/scripts/tests/feature-map.test.mjs @@ -0,0 +1,153 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { checkFeatureMap } from "../../../scripts/check-feature-map.mjs"; + +const recipe = `# Example + +A person can complete an example journey. + +## Sub-features + +- \`submit\`: saves the result and shows errors. + +## How to get to it (user POV) + +### \`task\` + +Open the task. + +## Driving it + +Preconditions: an isolated instance. + +### \`task\` + +Automated: [component test](../ui/src/pages/Example.test.tsx). + +Manual: submit, reload, and inspect the persisted result. + +## Gotchas + +- A component test does not prove live delivery. +`; + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), "feature-map-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const write = (path, text) => { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), text); + }; + const coverage = { version: 1, areas: [{ + id: "tasks", title: "Tasks", status: "partial", features: ["example.md"], + gap: "Task creation has no recipe yet.", paths: ["ui/src/pages/Example.tsx"], + }] }; + write("feature-map/README.md", "# Map\n\n## Features\n\n[Example](./example.md)\n"); + write("feature-map/example.md", recipe); + write("feature-map/coverage.json", JSON.stringify(coverage)); + write("ui/src/pages/Example.tsx", "export const Example = () => null;\n"); + write("ui/src/pages/Example.test.tsx", "// referenced test fixture\n"); + return { root, write, coverage, check: () => checkFeatureMap(root).errors }; +} + +test("the checked-in feature map matches the current repository", () => { + assert.deepEqual(checkFeatureMap().errors, []); +}); + +test("accepts a documented partial area and excludes tests and stories from page coverage", (t) => { + const f = fixture(t); + f.write("ui/src/pages/New.spec.tsx", ""); + f.write("ui/src/pages/New.stories.tsx", ""); + assert.deepEqual(f.check(), []); +}); + +test("new nested pages cannot hide under a previously covered directory", (t) => { + const f = fixture(t); + f.write("ui/src/pages/apps/new/NewPanel.tsx", ""); + assert.match(f.check().join("\n"), /unclassified page ui\/src\/pages\/apps\/new\/NewPanel.tsx/); +}); + +test("deleted or renamed test references fail even inside runnable commands", (t) => { + const f = fixture(t); + f.write("feature-map/example.md", recipe + "\n```sh\npnpm exec vitest run server/src/renamed.test.ts\n```\n"); + rmSync(join(f.root, "ui/src/pages/Example.test.tsx")); + const errors = f.check().join("\n"); + assert.match(errors, /missing test ui\/src\/pages\/Example.test.tsx/); + assert.match(errors, /missing test server\/src\/renamed.test.ts/); +}); + +test("broken documentation links fail, external URLs and local anchors are accepted", (t) => { + const f = fixture(t); + f.write("feature-map/example.md", recipe + "\n[External](https://example.com) [Here](#gotchas) [Missing](../doc/missing.md#section)\n"); + assert.deepEqual(f.check(), ["example.md: broken local link ../doc/missing.md#section"]); +}); + +test("references under .github are checked as full paths rather than scripts fragments", (t) => { + const f = fixture(t); + f.write(".github/scripts/tests/example.test.mjs", "// fixture\n"); + f.write("feature-map/example.md", recipe + "\n```sh\nnode --test .github/scripts/tests/example.test.mjs\n```\n"); + assert.deepEqual(f.check(), []); +}); + +test("new recipes must be indexed and missing recipes must be removed from the index", (t) => { + const f = fixture(t); + f.write("feature-map/unlisted.md", recipe); + f.write("feature-map/README.md", "# Map\n\n## Features\n\n[Missing](./missing.md)\n"); + const errors = f.check().join("\n"); + assert.match(errors, /Features must link example.md exactly once/); + assert.match(errors, /Features must link unlisted.md exactly once/); + assert.match(errors, /unknown recipe missing.md/); +}); + +test("missing sections and missing preconditions fail", (t) => { + const f = fixture(t); + f.write("feature-map/example.md", recipe.replace("## Gotchas", "## Notes")); + assert.match(f.check().join("\n"), /expected sections in order/); + f.write("feature-map/example.md", recipe.replace("Preconditions:", "Setup:")); + assert.match(f.check().join("\n"), /must start with Preconditions:/); +}); + +test("entry points cannot silently lack a driving recipe or a manual coverage statement", (t) => { + const f = fixture(t); + f.write("feature-map/example.md", recipe.replace("Open the task.", "Open the task.\n\n### `chat`\n\nOpen Chat.")); + assert.match(f.check().join("\n"), /entry points and driving recipe IDs must match/); + f.write("feature-map/example.md", recipe.replace("Manual:", "Notes:")); + assert.match(f.check().join("\n"), /describe Automated: evidence\/gaps and Manual: steps/); +}); + +test("duplicate entry-point and sub-feature IDs fail", (t) => { + const f = fixture(t); + f.write("feature-map/example.md", recipe.replace("Open the task.", "Open the task.\n\n### `task`\n\nAgain.") + .replace("## How to get", "- `submit`: another result.\n\n## How to get")); + const errors = f.check().join("\n"); + assert.match(errors, /unique H3 entry-point IDs/); + assert.match(errors, /unique backticked IDs/); +}); + +test("the inventory rejects stale paths, duplicate classification, and nonexistent recipes", (t) => { + const f = fixture(t); + f.coverage.areas.push({ ...f.coverage.areas[0], id: "other", features: ["missing.md"], + paths: ["ui/src/pages/Example.tsx", "ui/src/pages/Gone.tsx"] }); + f.write("feature-map/coverage.json", JSON.stringify(f.coverage)); + const errors = f.check().join("\n"); + assert.match(errors, /unknown recipe missing.md/); + assert.match(errors, /duplicate page ui\/src\/pages\/Example.tsx/); + assert.match(errors, /stale or non-page path ui\/src\/pages\/Gone.tsx/); +}); + +test("unmapped areas need an explanation and cannot claim recipe coverage", (t) => { + const f = fixture(t); + Object.assign(f.coverage.areas[0], { status: "unmapped", gap: " " }); + f.write("feature-map/coverage.json", JSON.stringify(f.coverage)); + const errors = f.check().join("\n"); + assert.match(errors, /explain the remaining gap/); + assert.match(errors, /unmapped areas must have none/); +}); + +test("CI continues to run the test glob that includes this gate", () => { + const workflow = readFileSync(new URL("../../workflows/pr-trusted.yml", import.meta.url), "utf8"); + assert.match(workflow, /node --test ['"]?\.github\/scripts\/tests\/\*\.test\.mjs/); +}); diff --git a/AGENTS.md b/AGENTS.md index c957ca41ead..00049317677 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,6 +152,12 @@ Notes: ## 7. Verification Before Hand-off +Use [feature-map/README.md](feature-map/README.md) to identify affected user +entry points and their verification recipes. Update the matching recipe when a +journey changes. Classify new UI page modules in `feature-map/coverage.json`, +either with a recipe or an explicit gap, and run `pnpm check:feature-map`. +The map records coverage scope, not proof that a live journey passed. + Default local/agent test path: ```sh diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 850081d3919..a41fdfbb0ad 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -419,6 +419,13 @@ npx paperclipai allowed-hostname dotta-macbook-pro ## Test Commands +Start from the [feature map](../feature-map/README.md) to find user entry points, +targeted tests, manual verification recipes, and explicit coverage gaps. When a +change adds or alters a journey, update its recipe and UI coverage inventory. +Run `pnpm check:feature-map` to catch stale links, missing test references, and +unclassified page modules. The existing PR quality-script test lane enforces the +same check; it does not claim that the documented journeys were driven live. + Use the cheap local default unless you are specifically working on browser flows: ```sh diff --git a/feature-map/README.md b/feature-map/README.md new file mode 100644 index 00000000000..e5fec0c0836 --- /dev/null +++ b/feature-map/README.md @@ -0,0 +1,118 @@ +# Paperclip feature map + +Start here when reproducing a user-facing bug, verifying a change, or deciding +which surfaces a fix must cover. Each recipe describes what a person can do, +where they can do it, the expected result, and the evidence needed to verify it. +Product behavior is still governed by [the implementation spec](../doc/SPEC-implementation.md). + +## Features + +| Feature | Use it to verify | +| --- | --- | +| [Questions and approvals](./questions-and-approvals.md) | Answering questions, reviewing plans, board approvals, and app-tool reviews across task, chat, and decision surfaces. | +| [Steering and queued messages](./steering.md) | Follow-ups during execution, queue edits, steering, interruption, and pause/resume. | +| [Connection setup](./connection-setup.md) | Catalog, link, task-request, and chat-channel setup; account access; interrupted setup and reconnection. | +| [Recovery](./recovery.md) | Understanding why work stopped, restoring a valid path, and recognizing when retry is unsafe or unavailable. | + +These four recipes are the **seed scope**, checked against source on 2026-10-05. +They are verification instructions, not a claim that every journey passed a live +test. A referenced component test proves that component; a scripted provider +proves the integration with that fixture. Neither proves a real provider login +or a model's behavior. Record executed results separately from this map. + +## Before driving a journey + +1. Use this checkout's [isolated test drive](../doc/DEVELOPING.md#one-command-isolated-manual-test-drive) + for manual product checks. It creates a temporary instance and prints its URL + and data directory. Do not guess that a server on port 3100 belongs to you. +2. Record the commit, URL, company, login role, relevant experimental settings, + adapter, runtime mode, and live versus simulated dependencies. Use disposable + tasks and provider resources. A fresh test drive creates a company and CEO, + but no task or first run. +3. Use the current navigation and company prefix. Paths in recipes omit that + prefix. Record whether the streamlined or production shell is selected; + Agent Chat, chat connectors, and the combined Inbox/Tasks view have separate + gates. A hidden surface is an unmet prerequisite, not a successful test. +4. Run the smallest relevant test first. Vitest commands below run from the + repository root after installing dependencies. Playwright recipes use their + named configuration and its isolated environment, not an unrelated running + instance. Reserve expensive runner/provider tests for the behavior at issue. +5. Exercise the actual user action, inspect its result, reload, and verify the + persisted state or continuation. Use read-only API checks to corroborate UI + evidence; creating state through the API does not prove the creation UI. + +For agent-driven acceptance work, the existing +[dev-workspace run/verify skill](../.agents/skills/paperclip-dev-workspace-run-verify-fix/SKILL.md) +and [evaluation skill](../.agents/skills/paperclip-evals/SKILL.md) describe runtime +ownership and evidence handling. Reuse them; this map introduces no environment +launcher, credentials, scheduled job, or second test framework. + +## Evidence contract + +Report each entry point as **passed**, **failed**, **blocked** (with its missing +prerequisite), or **not run**. Include the feature filename and entry-point ID, +commit/environment, user action, expected and observed result, command/exit code, +and evidence links. Screenshots or traces should show both the action and the +discriminating result. Include task/run/request IDs when relevant, without secrets. + +A fix is verified across its affected surfaces only when each has evidence or +an explicit reason it does not apply. Shared code alone does not prove parity. +Keep product regressions visible; do not change the recipe to bless a failure. +For Paperclip-assigned work, attach evidence through the +[artifact workflow](../doc/AGENT-ARTIFACTS.md). + +## Not yet mapped + +[The UI coverage inventory](./coverage.json) accounts for every non-test TSX +module under `ui/src/pages`, including supporting panels, legacy variants, and +labs. Entries are grouped by product area, with exact paths, linked recipes, +and a remaining gap. **Partial** means only the linked journeys are mapped. +**Unmapped** means there is no recipe yet. Neither is a runtime health verdict. + +The main backlog is onboarding and login; company, agent, project, and goal +management; task creation/search/list views; documents and artifacts; skills +and teams; budgets and activity; routines and pipelines; workspace management; +secrets, plugins, gateways, profiles, and instance administration. Chat setup +is seeded, but provider-by-provider messaging and attachment behavior is not. + +The page inventory is deliberately conservative: it includes helpers and unused +legacy modules rather than inferring reachability. It does **not** enumerate +every route, button, component, CLI command, API endpoint, provider, or harness +capability. CLI setup/administration and complete harness capability matrices +remain unmapped. New entry points inside an existing file still require review. + +## Keeping the map current + +```sh +pnpm check:feature-map +node --test .github/scripts/tests/feature-map.test.mjs +``` + +The checker uses only Node's standard library. The second command also runs in +the existing PR workflow's quality-script test glob, including UI-only and +documentation PRs. It checks: + +- Every feature recipe is linked from this index, with no stale feature links. +- Recipes contain a description and the four sections below, in order. +- Stable entry-point IDs have matching driving recipes, automated evidence + descriptions, and manual instructions or explicit gaps. +- Local Markdown links and repository test references exist. +- Every page module has exactly one inventory entry; removed paths, missing + recipes, empty gap explanations, and unclassified new files fail. + +When a PR changes a journey, update its entry points, recipe, and coverage gaps +in that PR. When it adds a page module, either map it or add an explicit gap to +the inventory. The gate checks structure and references; reviewers must still +check that the described behavior and tests match. No scheduled upkeep is enabled. + +Each recipe starts with an H1 and a user-visible description, then exactly: + +1. **Sub-features** — stable backticked IDs and observable behavior/states. +2. **How to get to it (user POV)** — one H3 backticked entry-point ID per surface. +3. **Driving it** — starts with `Preconditions:`; repeat each entry-point H3, + with `Automated:` and `Manual:` paragraphs. Name test scope and gaps honestly. +4. **Gotchas** — misleading look-alikes, feature gates, and invalid evidence. + +The format is inspired by +[Omnigent's feature map](https://github.com/omnigent-ai/omnigent/tree/91acfbbb59f6fc210ff95a9e9428aadd62e06582/feature-map). +Paperclip's recipes and checks follow its own task model and test infrastructure. diff --git a/feature-map/connection-setup.md b/feature-map/connection-setup.md new file mode 100644 index 00000000000..00c9328a9ae --- /dev/null +++ b/feature-map/connection-setup.md @@ -0,0 +1,150 @@ +# Connection setup + +People connect an app or account, choose who may use it, and recover incomplete +setup from either Apps or a task that needs access. A saved draft, a completed +provider login, a usable connection, and a resumed task are separate checkpoints. +Chat channels additionally require an endpoint and linked sender identity. + +## Sub-features + +- `catalog`: find a provider, inspect its available methods, and open setup; + unavailable methods explain their prerequisite rather than silently switching. +- `identity-and-access`: select personal/shared identity and eligible agents; + an empty explicit selection and company-wide access must not be confused. +- `credentials`: provider sign-in, key, or generic MCP URL follows its supported + setup method; credentials are not retained in browser draft storage. +- `handoff`: provider popup/full-page callback returns to the correct setup + host with the original task intent and identity choice intact. +- `draft-recovery`: cancel, Back, Save & exit, reload, Finish setup, and reconnect + preserve the exact draft rather than creating duplicate connections. +- `task-request`: only the addressed person resolves the requested connection; + usable access releases the pending task request through normal continuation. +- `channel-setup`: provider endpoint, represented agent, linked sender, and + communication instructions are configured before an external message is used. +- `failure`: denied login, unreachable/private URL, expired credentials, and + permission errors leave an actionable state rather than a false success. + +## How to get to it (user POV) + +### `apps-catalog` + +Open Apps (`/apps`), choose an app, and connect from its landing page. Select an +available method and access, then complete setup. Existing connections open +their details; unfinished ones offer a way to finish setup. + +### `mcp-link` + +Use **Connect with a link** or `/apps/byo` for a generic MCP server. Paste its +URL, select access, and follow the probe's sign-in/key guidance. Provider-specific +catalog setup and a generic endpoint may use different flows. + +### `task-request` + +Open the task's pending connection request and its setup action as the named +user. Setup can open in a task dialog and may use an OAuth popup. Follow the +request's original link if setup falls back to a standalone page. + +### `resume-connection` + +From Apps or connection details, use Finish setup, reconnect, or the offered +authentication recovery action. Reload or return from the provider before +checking whether the same saved connection became usable. + +### `chat-channel` + +With chat connectors enabled, select a supported chat/email provider in Apps +or open `/apps/chat/connect`. Complete that provider's endpoint setup, then +open its settings (`/apps/chat/:endpointId/settings`). Follow identity linking +as the intended sender before messaging the agent externally. + +## Driving it + +Preconditions: follow the [baseline](./README.md#before-driving-a-journey). +Choose a disposable provider account/resource and record its auth method, +deployment mode, account ownership, and access selection. For live setup, use +the provider's documented prerequisites in the +[connector playbook](../doc/connections/CONNECTOR-PLAYBOOK.md). Automated fixtures +do not establish OAuth registration, public callbacks, or production provider access. + +### `apps-catalog` + +Automated: [Browse](../ui/src/pages/apps/Browse.test.tsx), +[unconnected app](../ui/src/pages/apps/AppNotConnected.test.tsx), and +[AppsConnect](../ui/src/pages/apps/AppsConnect.test.tsx) cover catalog entry, +access selection, provider-specific method choices, and setup failures with +mocked APIs. [Connection intents](../tests/e2e/connection-intents.spec.ts) +drives store and task setup against one fake provider through a useful continuation: + +```sh +pnpm exec vitest run ui/src/pages/apps/Browse.test.tsx ui/src/pages/apps/AppsConnect.test.tsx +pnpm exec playwright test -c tests/e2e/playwright.config.ts tests/e2e/connection-intents.spec.ts +``` + +Manual: connect from an unconnected provider landing. Select the intended +identity and agent access, complete authentication, reload, and inspect the +saved connection. Run a permitted read-only action with an eligible agent and +verify its useful result. Try an ineligible agent and confirm denial. Repeat +with a user who cannot install company-wide; the UI must not imply broad access. + +### `mcp-link` + +Automated: AppsConnect's generic MCP cases cover probe-driven auth, private or +unreachable endpoints, preregistered clients, secret-backed headers, and unsafe +authorization URLs. These are component/API mocks, not remote provider proof. + +Manual: connect a known test MCP URL through the link flow, verify its saved +endpoint and a read-only action, then repeat with a bad URL and wrong credential. +The error should explain the next action and preserve non-secret setup context. +Test no-auth and authenticated endpoints separately; never infer one from the other. + +### `task-request` + +Automated: [interaction card](../ui/src/components/IssueThreadInteractionCard.test.tsx) +checks addressed-user controls. AppsConnect tests cover page/dialog parity, +task-bound OAuth, popup handoff, and verifying a saved connection before closing. +The connection-intents browser suite above exercises fixture-backed continuation. + +Manual: ask an agent to use an unavailable app, open its request as the named +user, and complete setup in the dialog. Verify the card settles, the exact task +resumes, and the agent returns a useful provider result. Repeat the page-host +fallback and denied OAuth path. As another user, confirm that merely seeing the +task does not grant the right to resolve the request. + +### `resume-connection` + +Automated: AppsConnect tests cover exact-draft resume, declined OAuth, reconnect +lookup failure/retry, response-lost creation, archived connection revival, and +saved account reuse. [App detail](../ui/src/pages/apps/AppDetail.test.tsx) covers +the connection host. These tests do not complete a real provider login. + +Manual: leave setup through its footer, reload Apps, choose Finish setup, and +confirm the same connection is resumed. Decline provider login and recover from +that checkpoint. Reconnect an expired disposable account and verify a permitted +action afterwards. Count connections before/after to catch accidental duplicates. + +### `chat-channel` + +Automated: [setup routing](../ui/src/pages/apps/chat/ChatEndpointSetup.routing.test.tsx), +[saved setup state](../ui/src/pages/apps/chat/ChatEndpointSetup.state.test.ts), +[email setup](../ui/src/pages/apps/chat/EmailEndpointSetup.test.tsx), and +[identity confirmation](../ui/src/pages/apps/chat/ChatIdentityConfirm.test.tsx) +cover shared UI logic with fixtures. Provider-specific live setup is not implied. + +Manual: complete setup for one named provider, reload endpoint settings, link +the intended sender, and send a disposable message. Verify one task and a useful +reply in the originating conversation. Inspect an unlinked sender's denial and +the endpoint's pause behavior. Repeat independently for each provider at issue; +Slack setup does not qualify Discord, Telegram, GitHub, email, or iMessage. + +## Gotchas + +- A successful OAuth callback is not proof of saved connection access or task + continuation. Verify all three checkpoints. +- Personal identity, shared account access, and per-agent access are distinct. + Connecting an account does not waive an Off action or Ask first review. +- A tool app connection is not a chat endpoint. The same provider can offer both. +- Old Apps/Connections URLs can redirect to Apps. Use the current navigation and + record the actual destination rather than documenting an obsolete screen. +- Full provider matrix, AI-account selection, gateway/profile administration, + and account revocation lifecycles remain outside this seed recipe. +- For governed actions after setup, use [questions and approvals](./questions-and-approvals.md). diff --git a/feature-map/coverage.json b/feature-map/coverage.json new file mode 100644 index 00000000000..eac27c176e7 --- /dev/null +++ b/feature-map/coverage.json @@ -0,0 +1,344 @@ +{ + "version": 1, + "areas": [ + { + "id": "tasks-and-attention", + "title": "Tasks, conversations, and attention", + "status": "partial", + "features": [ + "questions-and-approvals.md", + "steering.md", + "recovery.md" + ], + "gap": "Seed recipes cover interactions, queued follow-ups, and recovery. Creation, search, list filters, decision triage, documents, and complete conversation lifecycle still need recipes.", + "paths": [ + "ui/src/pages/AgentChat.tsx", + "ui/src/pages/AgentChats.tsx", + "ui/src/pages/BoardChat.tsx", + "ui/src/pages/DecisionQueuePage.tsx", + "ui/src/pages/Inbox.tsx", + "ui/src/pages/IssueDetail.tsx", + "ui/src/pages/Issues.tsx", + "ui/src/pages/LegacyInbox.tsx", + "ui/src/pages/MyIssues.tsx", + "ui/src/pages/Search.tsx", + "ui/src/pages/WhatNeedsMe.tsx" + ] + }, + { + "id": "agents", + "title": "Agents, organization, and runtime setup", + "status": "partial", + "features": [ + "recovery.md" + ], + "gap": "Recovery includes the run-detail surface. Hiring, model/account selection, configuration, files, org changes, and run history remain unmapped.", + "paths": [ + "ui/src/pages/AdapterManager.tsx", + "ui/src/pages/AgentDetail.production.tsx", + "ui/src/pages/AgentDetail.tsx", + "ui/src/pages/AgentToolsTab.tsx", + "ui/src/pages/Agents.production.tsx", + "ui/src/pages/Agents.tsx", + "ui/src/pages/NewAgent.tsx", + "ui/src/pages/Org.tsx", + "ui/src/pages/OrgChart.production.tsx", + "ui/src/pages/OrgChart.tsx" + ] + }, + { + "id": "apps", + "title": "App catalog and connection setup", + "status": "partial", + "features": [ + "connection-setup.md", + "questions-and-approvals.md" + ], + "gap": "Setup and app-tool review journeys are seeded. Complete provider/auth matrices, revocation, activity, advanced settings, and aggregator administration remain unmapped.", + "paths": [ + "ui/src/pages/apps/AggregatorAppManager.tsx", + "ui/src/pages/apps/AggregatorConnectDialog.tsx", + "ui/src/pages/apps/AppDetail.tsx", + "ui/src/pages/apps/AppLogo.tsx", + "ui/src/pages/apps/AppNotConnected.tsx", + "ui/src/pages/apps/AppsConnect.tsx", + "ui/src/pages/apps/AppsReview.tsx", + "ui/src/pages/apps/ArcadeDiscoverySetup.tsx", + "ui/src/pages/apps/Browse.tsx", + "ui/src/pages/apps/CatalogSourceFilters.tsx", + "ui/src/pages/apps/ComposioAppManager.tsx", + "ui/src/pages/apps/ComposioAppSetup.tsx", + "ui/src/pages/apps/ConnectionProvenanceChip.tsx", + "ui/src/pages/apps/Connections.tsx", + "ui/src/pages/apps/ExecutorManagementSetup.tsx", + "ui/src/pages/apps/PaperclipCloudOAuthHandoff.tsx", + "ui/src/pages/apps/ReviewQueueCard.tsx", + "ui/src/pages/apps/UnverifiedServerBadge.tsx", + "ui/src/pages/apps/app-detail/ActionTestDialog.tsx", + "ui/src/pages/apps/app-detail/ActivityPanel.tsx", + "ui/src/pages/apps/app-detail/AdvancedPanel.tsx", + "ui/src/pages/apps/app-detail/AgentConnectionAccess.tsx", + "ui/src/pages/apps/app-detail/BrowserUseSettingsPanel.tsx", + "ui/src/pages/apps/app-detail/ConnectedAggregatorApps.tsx", + "ui/src/pages/apps/app-detail/IdentitiesSection.tsx", + "ui/src/pages/apps/app-detail/PermissionsPanel.tsx", + "ui/src/pages/apps/app-detail/RailwayAccessPanel.tsx", + "ui/src/pages/apps/app-detail/ReviewPanel.tsx", + "ui/src/pages/apps/app-detail/SetupPanel.tsx", + "ui/src/pages/apps/connection-owner.tsx" + ] + }, + { + "id": "chat-channels", + "title": "Chat and email channel setup", + "status": "partial", + "features": [ + "connection-setup.md", + "questions-and-approvals.md", + "steering.md" + ], + "gap": "Shared setup, identity, and interaction entry points are seeded. Each provider needs separate live messaging, attachment, control, and delivery recipes.", + "paths": [ + "ui/src/pages/apps/chat/ChatCommunicationInstructions.tsx", + "ui/src/pages/apps/chat/ChatEndpointDetail.tsx", + "ui/src/pages/apps/chat/ChatEndpointSetup.tsx", + "ui/src/pages/apps/chat/ChatIdentityConfirm.tsx", + "ui/src/pages/apps/chat/EmailEndpointSetup.tsx", + "ui/src/pages/apps/chat/GitHubBotConfiguration.tsx", + "ui/src/pages/apps/chat/GitHubBotManagement.tsx", + "ui/src/pages/apps/chat/GitHubChatSetup.tsx", + "ui/src/pages/apps/chat/GitHubSetupPrompt.tsx", + "ui/src/pages/apps/chat/PhotonConnectStep.tsx", + "ui/src/pages/apps/chat/SetupPrompt.tsx", + "ui/src/pages/apps/chat/SlackAvatarStep.tsx", + "ui/src/pages/apps/chat/SlackIdentityStep.tsx", + "ui/src/pages/apps/chat/SlackSetupPrompt.tsx", + "ui/src/pages/apps/chat/SlackToolSettings.tsx" + ] + }, + { + "id": "skills-and-teams", + "title": "Skills, Skill Studio, and teams", + "status": "partial", + "features": [ + "questions-and-approvals.md" + ], + "gap": "Only the Skill Studio interaction surface is seeded. Authoring, imports, sources, tests, versions, policies, and team installation remain unmapped.", + "paths": [ + "ui/src/pages/CompanySkills.production.tsx", + "ui/src/pages/CompanySkills.tsx", + "ui/src/pages/SkillSources.tsx", + "ui/src/pages/SkillStudio.tsx", + "ui/src/pages/TeamCatalog.tsx", + "ui/src/pages/agent-skills/AgentSkillReleasePicker.tsx", + "ui/src/pages/agent-skills/AgentSkillRow.tsx", + "ui/src/pages/agent-skills/AgentSkillsTab.tsx", + "ui/src/pages/skills/ImportSkillsFromProjectDialog.tsx", + "ui/src/pages/skills/SkillImportProgress.tsx", + "ui/src/pages/skills/SkillPackagePreview.tsx", + "ui/src/pages/skills/SkillSourceTree.tsx" + ] + }, + { + "id": "projects-and-workspaces", + "title": "Projects and execution workspaces", + "status": "partial", + "features": [ + "recovery.md" + ], + "gap": "Recovery recipes describe workspace repair controls. Project/repository setup, workspace provisioning, services, files, and normal lifecycle remain unmapped.", + "paths": [ + "ui/src/pages/ExecutionWorkspaceDetail.tsx", + "ui/src/pages/ProjectDetail.tsx", + "ui/src/pages/ProjectWorkspaceDetail.tsx", + "ui/src/pages/Projects.tsx", + "ui/src/pages/Workspaces.tsx" + ] + }, + { + "id": "formal-approvals", + "title": "Formal governance approvals", + "status": "partial", + "features": [ + "questions-and-approvals.md" + ], + "gap": "Board list/detail decisions are mapped with service tests and manual recipes. Full browser coverage and every approval-type effect still need qualification.", + "paths": [ + "ui/src/pages/ApprovalDetail.tsx", + "ui/src/pages/Approvals.tsx" + ] + }, + { + "id": "auth-and-access", + "title": "Login, invitations, and access", + "status": "unmapped", + "features": [], + "gap": "No end-to-end login, bootstrap, invitation, membership, role, or session-expiry recipe yet.", + "paths": [ + "ui/src/pages/Auth.tsx", + "ui/src/pages/BoardClaim.tsx", + "ui/src/pages/CliAuth.tsx", + "ui/src/pages/CompanyAccess.tsx", + "ui/src/pages/InstanceAccess.tsx", + "ui/src/pages/InviteLanding.tsx", + "ui/src/pages/JoinRequestQueue.tsx", + "ui/src/pages/UserProfile.tsx" + ] + }, + { + "id": "company-and-instance", + "title": "Company and instance administration", + "status": "unmapped", + "features": [], + "gap": "Company creation/import/export, settings, experimental gates, profile, and environment administration need recipes.", + "paths": [ + "ui/src/pages/Companies.tsx", + "ui/src/pages/CompanyEnvironments.tsx", + "ui/src/pages/CompanyExport.tsx", + "ui/src/pages/CompanyImport.tsx", + "ui/src/pages/CompanySettings.tsx", + "ui/src/pages/InstanceExperimentalSettings.tsx", + "ui/src/pages/InstanceGeneralSettings.tsx", + "ui/src/pages/ProfileSettings.tsx" + ] + }, + { + "id": "goals-costs-and-activity", + "title": "Goals, dashboards, costs, and activity", + "status": "unmapped", + "features": [], + "gap": "Goal lifecycle, rollups, budget enforcement/settings, dashboards, timeline, and audit navigation need recipes.", + "paths": [ + "ui/src/pages/Costs.production.tsx", + "ui/src/pages/Costs.tsx", + "ui/src/pages/Dashboard.tsx", + "ui/src/pages/DashboardLive.tsx", + "ui/src/pages/GoalDetail.tsx", + "ui/src/pages/Goals.tsx", + "ui/src/pages/Timeline.tsx", + "ui/src/pages/audit/AuditFeed.production.tsx", + "ui/src/pages/audit/AuditFeed.tsx", + "ui/src/pages/audit/AuditHub.tsx", + "ui/src/pages/audit/AuditRuns.tsx", + "ui/src/pages/audit/CompanyActivity.production.tsx", + "ui/src/pages/audit/CompanyActivity.tsx", + "ui/src/pages/audit/RoutineAuditActivity.tsx" + ] + }, + { + "id": "artifacts", + "title": "Artifacts and work products", + "status": "unmapped", + "features": [], + "gap": "Upload, preview, versioned document review, download, and output handoff need recipes.", + "paths": [ + "ui/src/pages/Artifacts.tsx" + ] + }, + { + "id": "routines-and-pipelines", + "title": "Routines, pipelines, cases, and status cards", + "status": "unmapped", + "features": [], + "gap": "Schedules, webhooks, execution history, pipeline review, cases, and status-card refresh need recipes; experimental availability is separate.", + "paths": [ + "ui/src/pages/CaseDetail.tsx", + "ui/src/pages/Cases.tsx", + "ui/src/pages/PipelineSettings.tsx", + "ui/src/pages/Pipelines.tsx", + "ui/src/pages/RoutineDetail.production.tsx", + "ui/src/pages/RoutineDetail.tsx", + "ui/src/pages/Routines.production.tsx", + "ui/src/pages/Routines.tsx", + "ui/src/pages/StatusCards/ArchivedStatusCardRow.tsx", + "ui/src/pages/StatusCards/CreateStatusCardDialog.tsx", + "ui/src/pages/StatusCards/StatusCardDetailDrawer.tsx", + "ui/src/pages/StatusCards/StatusCardSettingsForm.tsx", + "ui/src/pages/StatusCards/StatusCardTile.tsx", + "ui/src/pages/StatusCards/SummarizerAgentSelect.tsx", + "ui/src/pages/StatusCards/index.tsx" + ] + }, + { + "id": "secrets", + "title": "Secrets and secret proposals", + "status": "unmapped", + "features": [], + "gap": "Company and user secret lifecycle, grants, proposals, vault import, and missing-secret recovery need recipes.", + "paths": [ + "ui/src/pages/Secrets.tsx", + "ui/src/pages/secrets/ImportFromVaultDialog.tsx", + "ui/src/pages/secrets/MissingUserSecretsBanner.tsx", + "ui/src/pages/secrets/MyUserSecretsTab.tsx", + "ui/src/pages/secrets/ProposalsTab.tsx", + "ui/src/pages/secrets/SecretPathName.tsx", + "ui/src/pages/secrets/SetMyUserSecretDialog.tsx", + "ui/src/pages/secrets/UserSecretDefinitionsTab.tsx", + "ui/src/pages/secrets/proposal-review.tsx", + "ui/src/pages/secrets/user-secret-presentation.tsx" + ] + }, + { + "id": "plugins-and-tools", + "title": "Plugins, gateways, and tool profiles", + "status": "unmapped", + "features": [], + "gap": "Plugin installation/configuration, gateways, token lifecycle, profiles, and tool administration need recipes.", + "paths": [ + "ui/src/pages/CompanySettingsPluginPage.tsx", + "ui/src/pages/PluginManager.tsx", + "ui/src/pages/PluginPage.tsx", + "ui/src/pages/PluginSettings.tsx", + "ui/src/pages/apps/gateways/ConnectClientDialog.tsx", + "ui/src/pages/apps/gateways/CopyableGatewayUrl.tsx", + "ui/src/pages/apps/gateways/EditGatewayDialog.tsx", + "ui/src/pages/apps/gateways/GatewayDetail.tsx", + "ui/src/pages/apps/gateways/GatewaysList.tsx", + "ui/src/pages/apps/gateways/NewGatewayDialog.tsx", + "ui/src/pages/apps/gateways/panels/AppsToolsPanel.tsx", + "ui/src/pages/apps/gateways/panels/GatewayActivityPanel.tsx", + "ui/src/pages/apps/gateways/panels/GatewayAdvancedPanel.tsx", + "ui/src/pages/apps/gateways/panels/OverviewPanel.tsx", + "ui/src/pages/apps/gateways/panels/TokensPanel.tsx", + "ui/src/pages/tools/AdvancedToolsRoute.tsx", + "ui/src/pages/tools/AuditTab.tsx", + "ui/src/pages/tools/GatewaysTab.tsx", + "ui/src/pages/tools/McpConfigHelpDialog.tsx", + "ui/src/pages/tools/PasteConfigTab.tsx", + "ui/src/pages/tools/ProfilesTab.tsx", + "ui/src/pages/tools/SmokeLabTab.tsx", + "ui/src/pages/tools/ToolsAccess.tsx", + "ui/src/pages/tools/connection-dialogs.tsx", + "ui/src/pages/tools/profiles/ProfileActionDialog.tsx", + "ui/src/pages/tools/profiles/ProfileDetail.tsx", + "ui/src/pages/tools/profiles/ProfileDetailRoute.tsx", + "ui/src/pages/tools/profiles/ProfileWizard.tsx", + "ui/src/pages/tools/profiles/ProfileWizardRoute.tsx", + "ui/src/pages/tools/profiles/ProfilesIndex.tsx", + "ui/src/pages/tools/profiles/ToolsAdminGate.tsx", + "ui/src/pages/tools/profiles/WizardToolsStep.tsx", + "ui/src/pages/tools/shared.tsx" + ] + }, + { + "id": "labs-and-fallbacks", + "title": "Developer labs and fallback pages", + "status": "unmapped", + "features": [], + "gap": "Design/lab/performance surfaces and not-found states are tracked for inventory completeness; they are not production journey proof.", + "paths": [ + "ui/src/pages/BootstrapSetupUxLab.tsx", + "ui/src/pages/CrossIssueCollaborationUxLab.tsx", + "ui/src/pages/DesignGuide.tsx", + "ui/src/pages/InviteUxLab.tsx", + "ui/src/pages/IssueChatLongThreadPerf.tsx", + "ui/src/pages/IssueChatUxLab.tsx", + "ui/src/pages/NotFound.tsx", + "ui/src/pages/ResponsibleUserDenialUxLab.tsx", + "ui/src/pages/RunTranscriptUxLab.tsx", + "ui/src/pages/SystemNoticeUxLab.tsx", + "ui/src/pages/TaskChatLab.tsx" + ] + } + ] +} diff --git a/feature-map/questions-and-approvals.md b/feature-map/questions-and-approvals.md new file mode 100644 index 00000000000..0446cd11c30 --- /dev/null +++ b/feature-map/questions-and-approvals.md @@ -0,0 +1,182 @@ +# Questions and approvals + +People answer an agent's questions and decide requests without losing the task's +history. A question, plan confirmation, governance approval, and app-tool review +can look similar but have different authority and continuation rules. Each +decision must stay attached to its original request and survive reload. + +## Sub-features + +- `questions`: single-choice, multiple-choice, custom, and free-text answers; + paged forms retain answers until submission and show validation failures. +- `dismiss-and-reopen`: unanswered questions remain in the feed after dismissal + or a newer message; reopening restores the form without inventing an answer. +- `confirmations`: accept, decline, or request changes; stale, expired, resolved, + and withdrawn requests cannot be acted on again. +- `plan-review`: inspect the requested plan revision before accepting or asking + for changes. An ordinary planning task can proceed to execution; Agent Chat + hands work to linked tasks and remains a conversation. +- `audience`: the current actor, company, named audience, and review policy govern + resolution. A visible request is not necessarily actionable by that viewer. +- `governance`: pending/history views, approve/reject, revision request, and + resubmission for formal board approvals. +- `tool-review`: approve once, always allow the displayed scope, or decline; + decision status and provider execution outcome remain distinct. +- `continuation`: one recorded response leads to the eligible waiting run or + continuation; repeated clicks, reloads, and competing tabs do not repeat work. + +## How to get to it (user POV) + +### `task-thread` + +Open a task from Tasks, search, a project, or its direct `/issues/:issueId` link. +Answer the question near the composer, or reopen its compact unanswered entry +in the history. Open the plan preview before using the confirmation controls. +Exercise both available task presentations when a change affects shared cards. + +### `agent-chat` + +With Agent Chat enabled, choose Chat and an agent (`/chats/:agentRef`). Answer, +dismiss, or reopen requests in this persistent conversation. A plan handoff here +creates linked work rather than turning the conversation into implementation. + +### `attention` + +Open Decisions (`/decisions`, including a saved queue) or an Inbox attention row. +Expand an actionable interaction inline; follow its task link to inspect context. +With combined Inbox/Tasks enabled, open the corresponding Tasks view instead. + +### `board-approval` + +Open Approvals → Pending or All (`/approvals/pending`, `/approvals/all`). Use the +card action or open `/approvals/:approvalId` for details, comments, revision +request, and resubmission. These are formal approvals, not ordinary questions. + +### `app-review` + +In a task, open **Review request** for an Ask first app call. The same request +appears in Apps → Review (`/apps/review`) and the connection's Review tab. +Use **Approve & run**, **Always allow**, or **Decline** on the surface under test. + +### `skill-test` + +Open Skill Studio, run a skill test that produces an interaction, and inspect +its output card. Skill-test authority is narrower than an ordinary company run. + +### `external-channel` + +In a configured chat channel, answer a published question or confirmation as a +linked sender. Open the associated Paperclip task to inspect the saved outcome. +Channel-specific forms and identity linking require separate provider evidence. + +## Driving it + +Preconditions: follow the [baseline](./README.md#before-driving-a-journey). +Use separate disposable requests for accept, decline, and revision paths. Record +the actor and request revision. Prepare a waiting question through a test agent; +a component fixture alone cannot prove live response delivery. + +### `task-thread` + +Automated: [interaction cards](../ui/src/components/IssueThreadInteractionCard.test.tsx), +[paged task cards](../ui/src/components/task-chat/TaskChatInteractionCard.test.tsx), +and [task thread](../ui/src/components/TaskChatThread.test.tsx) cover form, +receipt, and dismissal behavior. [Resolution routes](../server/src/__tests__/issue-thread-interaction-routes.test.ts) +and [response delivery](../server/src/__tests__/question-response-delivery.test.ts) +cover authority and durable continuation. Run a targeted set with: + +```sh +pnpm exec vitest run ui/src/components/TaskChatThread.test.tsx server/src/__tests__/question-response-delivery.test.ts +``` + +Manual: answer a multi-page question, go back, edit, then submit. Reload and +confirm the saved answers and one agent continuation. On a fresh request, +dismiss, reload, reopen, and verify the draft. Send a new comment while a +question remains unanswered; it must stay reopenable in history. Try a denied +actor and a request already resolved in another tab; expect an explicit error +or settled receipt, never a second decision. For plans, change the plan revision +before trying the old approval, then review the current revision. + +### `agent-chat` + +Automated: the [task thread suite](../ui/src/components/TaskChatThread.test.tsx) +parameterizes unanswered-question history for conversation mode on and off. +This proves component parity, not a model's plan handoff. + +Manual: repeat dismissal, reload, reopening, and answer submission in Chat. +Approve a plan and inspect the linked execution task and copied plan. The chat +history remains and the conversation does not close as a completed task. + +### `attention` + +Automated: [attention resolver](../ui/src/components/AttentionInteractionResolver.test.tsx) +covers preparing-request refresh without accepting it. The resolution-route +tests above cover the API, not every Decisions or Inbox navigation path. + +Manual: open the same pending request in an attention row and task tab. Resolve +from the row; both surfaces must settle after refresh. Check an empty queue, +loading, permission denial, and a stale request. Repeat with combined Inbox/Tasks +on when that surface is part of the change. Full navigation parity remains a +manual coverage gap. + +### `board-approval` + +Automated: [approval service](../server/src/__tests__/approvals-service.test.ts) +and [idempotency routes](../server/src/__tests__/approval-routes-idempotency.test.ts) +cover persistence and duplicate decisions. They do not drive the approval pages. + +Manual: create disposable formal approval requests; approve one from the list, +reject another from details, and request revision on a third. Resubmit that +third request and verify Pending/All and detail history after reload. Confirm +the recorded decision and its governed effect, not just a success toast. + +### `app-review` + +Automated: [connection reviews](../tests/e2e/connection-reviews.spec.ts) drives +task/queue synchronization, approve, decline, remembered permission, provider +failure, and restart with a scripted process agent and local MCP provider: + +```sh +pnpm exec playwright test -c tests/e2e/connection-reviews.config.ts +``` + +Manual: dismiss and reopen a review, decide it from each entry point using fresh +requests, and inspect the continuation's useful result. Always allow must show +its scope; a future action with changed arguments should obey the saved rule. +Provider failure must remain an approved decision with a failed execution. +Real provider and model checks follow [Task reviews](../doc/connections/TASK-REVIEWS.md#verification-workflows). + +### `skill-test` + +Automated: [read-only interaction contract](../server/src/__tests__/issue-interactions-read-only-contract.test.ts) +checks that fetching interactions does not mutate them. Shared card suites cover +rendering. Neither proves the complete Skill Studio host or its permission matrix. + +Manual: run a skill test with a question, answer through its visible output, +and inspect the test result. Attempt a governed action outside the test's scope; +expect denial. The complete Skill Studio browser journey is not covered by the +referenced unit tests. + +### `external-channel` + +Automated: [chat question forms](../server/src/services/chat-question-forms.test.ts), +[Discord forms](../server/src/services/chat-discord-question-forms.test.ts), +and [interaction publications](../server/src/__tests__/chat-interaction-publications.test.ts) +cover serialization and delivery logic with fixtures. + +Manual: use a disposable provider conversation, answer as the linked sender, +and check the task's persisted result and one continuation. Repeat from an +unlinked or wrong sender and with an expired card. A browser answer does not +qualify Slack, Discord, Telegram, email, or iMessage; record each as not run +unless its actual provider journey was driven. + +## Gotchas + +- Dismissing a question does not answer it. Dismissing an app review neither + grants permission nor resumes the agent. Ordinary comments cannot approve it. +- A resolved request and a successful tool execution are separate facts. +- Do not reuse an accepted request to test rejection; use another request. +- Agent Chat and skill tests have distinct lifecycle/authority rules despite + sharing UI. A watchdog cannot acquire board authority by resolving a card. +- An answer to an old question can queue behind a successor run; it does not + automatically steer that run. See [steering](./steering.md). diff --git a/feature-map/recovery.md b/feature-map/recovery.md new file mode 100644 index 00000000000..b5c8b60eb9c --- /dev/null +++ b/feature-map/recovery.md @@ -0,0 +1,153 @@ +# Recovery + +People can see why a task stopped, who owns the next action, and whether a safe +continuation is available. Recovery retains task context and ownership, honors +pause and budget gates, and distinguishes confirmed failure from an uncertain +external outcome. A retry acknowledgement alone is not proof that work resumed. + +## Sub-features + +- `diagnosis`: a task/run exposes the recorded cause, evidence, owner, attempt + progress, and next action without requiring a raw-log interpretation. +- `bounded-continuation`: supported interrupted work continues with preserved + conversation context and bounded attempts, without blindly replaying tools. +- `source-action`: source-scoped recovery actions remain attached to the task; + independent repair work may have its own issue and return owner. +- `exhaustion`: exhausted or overdue recovery exposes the current gate and + permitted operator action; it does not claim an automatic path is still live. +- `workspace`: workspace divergence/export failures show the applicable repair, + reconcile, or isolated reissue action with evidence and authority checks. +- `unknown-outcome`: uncertain external side effects require inspection and + an explicit decision, not automatic replay. +- `user-continuation`: saved user messages after a stop are reconsidered once + cleanup proves the old execution is stopped; duplicate wakes are avoided. +- `reconnect`: browser stream recovery refreshes persisted state without claiming + the agent itself restarted or that a failed run succeeded. + +## How to get to it (user POV) + +### `source-task` + +Open the affected task from Tasks, Inbox/Blocked, search, or a direct task link. +Inspect the recovery notice/card in its history, expand details if needed, and +use the currently offered action. Follow any linked repair task and return owner. + +### `run-detail` + +Open the agent's run (`/agents/:agentId/runs/:runId`) from activity or the task's +run link. Inspect failure/workspace recovery there, then return to the source +task. This is a separate host for shared recovery controls. + +### `operator-retry` + +On an exhausted disposition or workspace recovery card, inspect the current +reason and use the permitted retry/repair/reconcile action. Some repairs require +confirmation; unavailable actions must explain the gate or remain absent. + +### `post-stop-message` + +After a stop, send a new instruction on the same task or inspect one already +saved while cleanup was pending. Observe the waiting message, continuation, and +eventual result rather than manually changing the task status to look complete. + +### `browser-reconnect` + +While viewing an active or recovering task, interrupt the browser connection +and restore it, or suspend and return to the tab. Reload is a separate check. +The current task and run state should reconcile with the server. + +## Driving it + +Preconditions: follow the [baseline](./README.md#before-driving-a-journey). +Use a disposable instance and reproducible failure. Record source task/run, +adapter/runtime mode, cause, attempt count, current owner, and verified stop +evidence. Do not kill an unrelated server or induce provider side effects just +to get a failure screen. Use the existing deterministic recovery suite first. + +### `source-task` + +Automated: [recovery actions](../server/src/__tests__/issue-recovery-actions.test.ts) +covers source scoping, bounded attempts, operator stops, quota monitoring, +ownership, stale action retirement, and continuation deduplication. +[Recovery cards](../ui/src/components/IssueRecoveryActionCard.test.tsx) cover +diagnosis, retry progress, exhaustion, and action availability. The fixture-backed +[execution-recovery journeys](../tests/e2e/execution-recovery/recovery.spec.ts) +start their own test-drive instances and exercise safe, uncertain, restart, +manager-lineage, and legacy-unknown paths: + +```sh +pnpm exec vitest run server/src/__tests__/issue-recovery-actions.test.ts ui/src/components/IssueRecoveryActionCard.test.tsx +pnpm exec playwright test -c tests/e2e/execution-recovery/playwright.config.ts recovery.spec.ts +``` + +Manual: open a stopped task through each affected navigation path. Confirm its +explanation, original owner, recovery owner if different, and next action. Follow +the supported recovery until the source task produces a useful result or a +specific escalation. Reload and verify that resolved notices retire and attempt +history remains. A fixture pass does not qualify every live harness/provider. + +### `run-detail` + +Automated: [run workspace recovery](../ui/src/components/RunWorkspaceRecoverySurface.test.tsx) +tests this host; it is component coverage, not an end-to-end repair. + +Manual: open the exact failed run and compare its recovery diagnosis to the +source task. Use an eligible action from the run surface, return to the task, +and verify the same action state, persisted outcome, and restored execution. +Repeat with an actor who lacks authority. Full run-detail navigation remains a +manual gap. + +### `operator-retry` + +Automated: [disposition notices](../ui/src/components/DispositionRecoveryNotice.test.tsx) +tests pending/error/gated states and waiting for the real retry acknowledgement. +The recovery-card suite covers workspace divergence, reissue, reconcile, and +confirmation. [Workspace export](../ui/src/components/WorkspaceExportRecovery.test.tsx) +covers its separate repair surface. + +Manual: reproduce an exhausted disposable action. Click once, verify pending +state prevents repeated clicks, and inspect a successful continuation or an +explicit rejection. Reload to confirm the durable outcome. For a workspace +repair, record the compared branches/revisions and preserve uncommitted work; +confirm the repair's actual result before resuming. Check the denied and stale +action paths. A changed card label is insufficient proof of workspace integrity. + +### `post-stop-message` + +Automated: [legacy continuation](../server/src/services/recovery/legacy-continuation.test.ts), +[queue routes](../server/src/__tests__/issue-queued-comments-routes.test.ts), and +[native restart recovery](../server/src/services/native-runtime/native-restart-recovery.test.ts) +cover their respective recovery paths. [Legacy failure continuation](../tests/e2e/legacy-failure-continuation.spec.ts) +adds a browser journey; it is not native runtime proof. + +Manual: stop a disposable run, submit a distinguishable follow-up, and observe +any cleanup wait. After confirmed stop, verify the follow-up reaches one eligible +continuation and produces the requested answer. Repeat with a task/ancestor +pause and a budget hold: recovery must not bypass the gate. For unknown external +outcomes, inspect the provider before authorizing another operation. + +### `browser-reconnect` + +Automated: [live-update recovery](../ui/src/context/LiveUpdatesProvider.recovery.test.tsx) +covers client transport recovery with mocks. It does not prove runner restart. + +Manual: go offline during a disposable run, let the server reach a new state, +then reconnect. Confirm task/run/attention data refresh without duplicate +comments or decisions; reload and compare. A healthy websocket proves transport +only. Separately inspect the underlying execution if it is still stuck. + +## Gotchas + +- Legacy and native runtimes have different ownership and restart paths. Record + which one was exercised instead of inferring coverage from the adapter name. +- A process disappearance, elapsed timeout, or successful API retry does not + prove an external tool operation never happened. Unknown outcomes stay unknown + until corroborated; automatic replay can duplicate a side effect. +- Active-run watchdogs, task watchdogs, reviewers, and recovery owners have + different authority. No recovery card grants board permission by itself. +- Idle Agent Chat is not stranded work. An intentional operator stop must not + silently trigger recovery that fights the user's instruction. +- Source inspection and mocked component tests do not qualify real server + restart continuity. Use the appropriate runtime suite for that claim. +- See [execution semantics](../doc/execution-semantics.md) for the governing + recovery contract and [steering](./steering.md) for saved follow-ups. diff --git a/feature-map/steering.md b/feature-map/steering.md new file mode 100644 index 00000000000..02d1cfa91bc --- /dev/null +++ b/feature-map/steering.md @@ -0,0 +1,135 @@ +# Steering and queued messages + +People can send follow-ups while an agent works, manage saved messages that have +not been consumed, and choose when to steer or interrupt. Stopping a run and +pausing a task are separate controls. The UI must preserve the message, explain +why it is waiting, and only claim delivery after an authoritative acknowledgement. + +## Sub-features + +- `queue`: a busy task saves follow-ups and shows their order and wait reason. +- `edit-order-discard`: edit full Markdown, reorder, and discard eligible saved + messages; ownership, revision conflicts, and already-dispatched states matter. +- `steer`: explicit user action sends a queued message into a compatible active + run; unavailable capabilities and provider rejection remain visible. +- `interrupt`: stop the current execution and carry accepted follow-ups into + the eligible continuation without delivering them twice. +- `stop-and-pause`: distinguish stop-in-flight, confirmed stop, task/ancestor + pause, agent pause, and budget hold. Resume must respect the actual gate. +- `drafts`: switching tasks, reloading, and uncertain submission retain the + correct task's draft and attachment receipts. +- `approval-queue`: answered questions and accepted approvals have durable + delivery distinct from editable ordinary comments. + +## How to get to it (user POV) + +### `task-composer` + +Open an assigned task (`/issues/:issueId`). Start work, then send follow-ups while +the agent runs. Use the queued-message controls to edit, reorder, discard, +steer, or interrupt when offered. Use the composer stop/pause control separately. +Both task presentations need attention when changing their shared behavior. + +### `agent-chat` + +Enable Agent Chat and open an agent conversation. Send another message during +a reply and inspect the shared queue. `/new` starts fresh provider context in +the same conversation; it is an ordered boundary, not a normal model prompt. + +### `request-response` + +Answer an earlier question or accept a confirmation while successor work is +active on the task. Inspect the resulting queued response and its available +steer/interrupt control instead of assuming acceptance interrupted execution. + +### `external-channel` + +Send a follow-up from a linked external chat conversation while its task is +running. Use that provider's supported stop/control affordance where available; +inspect the resulting task and run in Paperclip. + +## Driving it + +Preconditions: follow the [baseline](./README.md#before-driving-a-journey). +Use an agent that stays busy long enough to inspect the queue. Record the adapter, +native/legacy mode, and advertised steering capability. Test supported and +unsupported steering without treating an absent capability as a product failure. + +### `task-composer` + +Automated: [queued-message UI](../ui/src/components/task-chat/TaskChatQueuedMessages.test.tsx) +covers order, failure restoration, unavailable steering, stale revisions, and +discard acknowledgements. [Queue routes](../server/src/__tests__/issue-queued-comments-routes.test.ts) +cover durable ordering, authorization, steering acknowledgements, interrupt +recovery, and concurrent dispatch. [Composer](../ui/src/components/task-chat/TaskChatComposer.test.tsx) +covers drafts and uncertain saves. + +```sh +pnpm exec vitest run ui/src/components/task-chat/TaskChatQueuedMessages.test.tsx server/src/__tests__/issue-queued-comments-routes.test.ts +``` + +Automated stop coverage is separate: +[composer stop](../tests/e2e/composer-stop.spec.ts) with +[its configuration](../tests/e2e/playwright-composer-stop.config.ts) and +[ACP stop/continuation](../tests/e2e/acp-stop-continuation.spec.ts). Inspect their +fixture and opt-in prerequisites; skipped native cases are not native proof. + +Manual: queue three distinguishable messages. Edit one, reorder them, reload, +and confirm the saved order. Discard one and confirm it never reaches execution. +Steer one while busy and verify the agent receives that exact message once. +Repeat with a rejected steer and a stale second-tab edit; the UI must preserve +retryable content and explain the conflict. On a fresh run, interrupt with a +queued follow-up and verify confirmed stop followed by one continuation. Pause +the task (and separately its ancestor), preserve a draft, then resume the actual +hold; no hidden send should occur while paused. Check both task presentations +when available; the named component tests do not prove every host integration. + +### `agent-chat` + +Automated: [agent chat sessions](../tests/e2e/agent-chat-sessions.spec.ts) covers +persistent conversation/session behavior with deterministic harness fixtures; +shared composer tests cover the controls. These do not qualify live steering +for every harness. + +Manual: queue two follow-ups while the conversation replies. Reload, edit or +discard one, then let the other run. Verify one reply and unchanged conversation +identity. Send `/new`, then a new prompt; old history remains visible, and the +new turn uses fresh provider context. Verify a useful answer rather than only +the session-divider UI. Repeat the queue operations with a live capable harness +when the change touches delivery. + +### `request-response` + +Automated: [response delivery](../server/src/__tests__/question-response-delivery.test.ts) +proves queueing/coalescing and retry-safe receipts. The queue-route suite tests +separate comment and approval queues, explicit steering, and fresh-session +restrictions on plan approvals. + +Manual: answer an old question during a new run, confirm that it is saved and +queued, and let the active run end. Verify exactly one continuation with the +answer. Repeat an eligible approval with explicit steer; another approval that +requires a fresh session must not offer same-turn delivery. Do not edit an +approval outcome through an ordinary queued-message editor. + +### `external-channel` + +Automated: [chat interaction arbitration](../server/src/services/chat-interaction-arbitration.test.ts) +covers backend arbitration with simulated requests. It is not a provider UI test. + +Manual: send two distinguishable messages through one disposable linked +conversation while busy. Inspect their saved task order, run delivery, and +external reply. Check a duplicate provider event and unauthorized sender. +Provider-specific control semantics and real transport timing remain a manual +gap; do not assume browser queue controls are available in every channel. + +## Gotchas + +- Paperclip's queue is server-owned once saved; it is not Omnigent's unsent + browser draft buffer. A stale edit must not overwrite a consumed message. +- Steer availability depends on the active runtime's capability, not its brand + name. A UI row disappearing optimistically does not prove provider receipt. +- Interrupt, stop, task pause, agent pause, and budget pause have different + effects. A running indicator disappearing does not prove remote work stopped. +- An interrupted run may need verified cleanup before a saved follow-up can + start. Preserve and inspect the wait reason; see [recovery](./recovery.md). +- A healthy idle Agent Chat is waiting for input, not stranded unfinished work. diff --git a/package.json b/package.json index 1d7fe58f88d..2247b98a934 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "check:node-version": "node scripts/check-node-version-policy.mjs", "check:no-git-push": "node scripts/check-no-git-push.mjs", "check:module-boundaries": "node scripts/check-module-boundaries.mjs", + "check:feature-map": "node scripts/check-feature-map.mjs", "test:check-no-git-push": "node --test scripts/check-no-git-push.test.mjs", "test:install-sh-docker": "./scripts/test-install-sh-docker.sh", "test:hermes-gateway-smoke": "node --test scripts/smoke/hermes-gateway-smoke.test.mjs", diff --git a/scripts/check-feature-map.mjs b/scripts/check-feature-map.mjs new file mode 100644 index 00000000000..0c2bc791039 --- /dev/null +++ b/scripts/check-feature-map.mjs @@ -0,0 +1,178 @@ +#!/usr/bin/env node + +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +const defaultRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const sections = [ + "Sub-features", + "How to get to it (user POV)", + "Driving it", + "Gotchas", +]; + +function walk(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = resolve(directory, entry.name); + return entry.isDirectory() ? walk(path) : entry.isFile() ? [path] : []; + }).sort(); +} + +function withoutFences(text) { + return text.replace(/^(`{3,}|~{3,})[^\n]*\n[\s\S]*?^\1\s*$/gm, ""); +} + +function entryIds(text, label, errors) { + const headings = [...text.matchAll(/^### (.+)$/gm)].map((match) => match[1]); + const ids = headings.map((heading) => /^`([a-z][a-z0-9-]*)`$/.exec(heading)?.[1]); + if (!ids.length || ids.some((id) => !id) || new Set(ids).size !== ids.length) { + errors.push(`${label}: use unique H3 entry-point IDs, e.g. ### \`task-thread\``); + } + return ids.filter(Boolean); +} + +export function checkFeatureMap(repoRoot = defaultRoot) { + const errors = []; + const mapRoot = resolve(repoRoot, "feature-map"); + const read = (path) => readFileSync(resolve(repoRoot, path), "utf8"); + const normalize = (path) => relative(repoRoot, path).split(sep).join("/"); + const isFile = (path) => existsSync(path) && statSync(path).isFile(); + const localFile = (base, path) => { + const target = resolve(base, path); + const rel = relative(repoRoot, target); + return rel !== ".." && !rel.startsWith(`..${sep}`) && isFile(target); + }; + + const documents = readdirSync(mapRoot).filter((file) => file.endsWith(".md")).sort(); + const recipes = documents.filter((file) => file !== "README.md"); + if (!recipes.length) errors.push("feature-map: expected at least one recipe"); + const index = read("feature-map/README.md"); + const featureSection = index.split("\n## Features\n")[1]?.split("\n## ")[0] ?? ""; + const indexed = [...featureSection.matchAll(/\]\(\.\/([\w-]+\.md)\)/g)].map((m) => m[1]); + for (const file of recipes) { + if (indexed.filter((link) => link === file).length !== 1) { + errors.push(`README.md: Features must link ${file} exactly once`); + } + } + for (const file of indexed) { + if (!recipes.includes(file)) errors.push(`README.md: unknown recipe ${file}`); + } + + let entryPointCount = 0; + for (const file of documents) { + const raw = read(`feature-map/${file}`); + const text = withoutFences(raw); + // Map documents use inline Markdown links. Anchors and external links are + // allowed; local targets must be existing files inside the checkout. + for (const [, target] of text.matchAll(/\]\(([^)\s]+)\)/g)) { + const path = target.split("#")[0]; + if (!path || /^[a-z][a-z0-9+.-]*:/i.test(path)) continue; + if (!localFile(mapRoot, path)) errors.push(`${file}: broken local link ${target}`); + } + // Also validate test paths in runnable code blocks or backticked references. + // This deliberately checks filenames, not test titles or behavioral coverage. + const testRefs = [...raw.matchAll(/(? match[1]); + for (const path of new Set(testRefs)) { + if (!localFile(repoRoot, path)) errors.push(`${file}: missing test ${path}`); + } + if (file === "README.md") continue; + if (!/^# [^\n]+\n\s*\n\S/.test(text)) { + errors.push(`${file}: start with an H1 and a description paragraph`); + } + const headings = [...text.matchAll(/^## (.+)$/gm)].map((match) => match[1]); + if (JSON.stringify(headings) !== JSON.stringify(sections)) { + errors.push(`${file}: expected sections in order: ${sections.join(", ")}`); + continue; + } + const bodies = sections.map((title) => text.split(`\n## ${title}\n`)[1]?.split("\n## ")[0] ?? ""); + if (bodies.some((body) => !body.trim())) errors.push(`${file}: sections must not be empty`); + const subIds = [...bodies[0].matchAll(/^- `([a-z][a-z0-9-]*)`:/gm)].map((match) => match[1]); + if (!subIds.length || new Set(subIds).size !== subIds.length) { + errors.push(`${file}: Sub-features needs unique backticked IDs`); + } + const entries = entryIds(bodies[1], `${file} entry points`, errors); + const drives = entryIds(bodies[2], `${file} driving recipes`, errors); + entryPointCount += entries.length; + if (JSON.stringify([...entries].sort()) !== JSON.stringify([...drives].sort())) { + errors.push(`${file}: entry points and driving recipe IDs must match`); + } + if (!bodies[2].trimStart().startsWith("Preconditions:")) { + errors.push(`${file}: Driving it must start with Preconditions:`); + } + for (const id of drives) { + const recipe = bodies[2].split(`### \`${id}\`\n`)[1]?.split("\n### ")[0] ?? ""; + if (!/^Automated: \S/m.test(recipe) || !/^Manual: \S/m.test(recipe)) { + errors.push(`${file}#${id}: describe Automated: evidence/gaps and Manual: steps`); + } + } + if (!testRefs.length) errors.push(`${file}: reference at least one existing test file`); + } + + const inventory = JSON.parse(read("feature-map/coverage.json")); + if (inventory.version !== 1 || !Array.isArray(inventory.areas) || !inventory.areas.length) { + errors.push("coverage.json: expected version 1 and nonempty areas"); + return { errors, recipes: recipes.length, entryPoints: entryPointCount, pages: 0 }; + } + const pages = walk(resolve(repoRoot, "ui/src/pages")) + .filter((path) => path.endsWith(".tsx") && !/\.(test|spec|stories)\.tsx$/.test(path)) + .map(normalize); + if (!pages.length) errors.push("coverage.json: expected UI page modules"); + const accounted = new Set(); + const areaIds = new Set(); + for (const area of inventory.areas) { + if (!area || typeof area !== "object") { + errors.push("coverage.json: each area must be an object"); + continue; + } + const label = `coverage.json: ${area.id ?? "unnamed area"}`; + if (typeof area.id !== "string" || !/^[a-z][a-z0-9-]*$/.test(area.id) || areaIds.has(area.id)) { + errors.push(`${label}: expected a unique stable area ID`); + } + areaIds.add(area.id); + if (typeof area.title !== "string" || !area.title.trim()) errors.push(`${label}: title required`); + if (!["mapped", "partial", "unmapped"].includes(area.status)) errors.push(`${label}: invalid status`); + if (area.status !== "mapped" && (typeof area.gap !== "string" || !area.gap.trim())) { + errors.push(`${label}: explain the remaining gap`); + } + if (!Array.isArray(area.features)) { + errors.push(`${label}: features must be an array`); + } else { + if ((area.status === "unmapped") !== (area.features.length === 0)) { + errors.push(`${label}: mapped/partial areas need recipes; unmapped areas must have none`); + } + for (const feature of area.features) { + if (!recipes.includes(feature)) errors.push(`${label}: unknown recipe ${feature}`); + } + } + if (!Array.isArray(area.paths) || !area.paths.length) { + errors.push(`${label}: paths must be a nonempty array`); + continue; + } + for (const path of area.paths) { + if (!pages.includes(path)) errors.push(`${label}: stale or non-page path ${path}`); + if (accounted.has(path)) errors.push(`${label}: duplicate page ${path}`); + accounted.add(path); + } + } + for (const path of pages) { + if (!accounted.has(path)) errors.push(`coverage.json: unclassified page ${path}; map it or record a gap`); + } + return { errors, recipes: recipes.length, entryPoints: entryPointCount, pages: pages.length }; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const result = checkFeatureMap(); + if (result.errors.length) { + console.error(result.errors.join("\n")); + process.exitCode = 1; + } else { + console.log(`Feature map OK: ${result.recipes} recipes, ${result.entryPoints} entry points, ${result.pages} page modules accounted for (not live verification).`); + } + } catch (error) { + console.error(`Feature map check failed: ${error.message}`); + process.exitCode = 1; + } +}