diff --git a/doc/connections/CONNECTOR-PLAYBOOK.md b/doc/connections/CONNECTOR-PLAYBOOK.md index 04c650f7f11..55b54fb27e1 100644 --- a/doc/connections/CONNECTOR-PLAYBOOK.md +++ b/doc/connections/CONNECTOR-PLAYBOOK.md @@ -5,6 +5,13 @@ and shipping Paperclip app connections. Status: canonical end-to-end authoring guide for Apps v2 catalog connections. +Experimental voice connections use the same company-scoped channel endpoints, +vault, task bindings, delivery ledger and publications, with `transport: voice` +and an explicit runtime rather than Chat SDK. Do not expose voice transport as +an MCP tool connection. Speko's browser implementation and its remaining phone +qualification gate are documented in the [native voice report](../plans/2026-10-09-speko-pr-qualification.md) +and [E2E runbook](../plans/2026-09-11-speko-voice-e2e-runbook.md). The optional +Company Phone Agent is a catalog template; selecting it must remain optional. For connector artwork, follow [Connector icons](./CONNECTOR-ICONS.md): fixed gray Paperclip frames, authentic vendor artwork, explicit theme variants, optical fit and exact provenance. Brand-library additions do not activate connectors. Use the shared registry/resolver and branding generator; do not introduce per-screen logos or outer-surface overrides. This runbook is the repeatable, agent-executable procedure for adding a vendor diff --git a/doc/plans/2026-09-11-speko-storybook-inventory.md b/doc/plans/2026-09-11-speko-storybook-inventory.md new file mode 100644 index 00000000000..892bd0936f9 --- /dev/null +++ b/doc/plans/2026-09-11-speko-storybook-inventory.md @@ -0,0 +1,95 @@ +# Speko component-to-story coverage inventory + +This document is a dated qualification snapshot. The current catalog is generated from each story’s `parameters.paperclip` annotation; this report is not a separate catalog registry. + +Updated: 2026-10-13. Speko remains experimental. The source inventory contains **157 production-component stories**. The prior 148-story build passed 592 interaction/accessibility checks during PR preparation; 188 affected history/page checks passed, including the two secondary-error stories. The final 153-story build passed, and all 220 affected page, provider-setup, task-launcher and incoming-call interaction/accessibility checks passed (`/tmp/speko-pr-storybook-all-review-checks.log`). The narrow-light and desktop-dark ConfigureProvider captures were inspected after the footer change; canonical Linux baseline publication is still outstanding. Setup now selects existing agents without an installer. `ChooseAgent`, `ChooseAgentLongName`, and `ChooseAgentEmpty` cover selection, keyboard use, disabled Continue, empty inventory and focus restoration. The changed setup subset passed **20 interaction/accessibility checks** across light/dark themes and 390/1200-pixel viewports with reduced motion (`/tmp/speko-existing-agent-stories.log`). The October 7 full-suite results below are historical; removed installer stories no longer apply. Fixtures block external requests and real microphone access. + +| Story module | Story IDs | +| --- | --- | +| `./storybook/stories/chat-channels/speko/speko-call-provider.stories.tsx` | `connections-speko-call-provider--initial`, `connections-speko-call-provider--active`, `connections-speko-call-provider--navigation`, `connections-speko-call-provider--new-conversation`, `connections-speko-call-provider--company-change`, `connections-speko-call-provider--sign-out`, `connections-speko-call-provider--end-and-restart`, `connections-speko-call-provider--long-conversation`, `connections-speko-call-provider--restart-in-panel` | +| `./storybook/stories/chat-channels/speko/speko-connection-status.stories.tsx` | `connections-speko-connection-status--attention`, `connections-speko-connection-status--revoked`, `connections-speko-connection-status--removed`, `connections-speko-connection-status--paused` | +| `./storybook/stories/chat-channels/speko/speko-native-conversation.stories.tsx` | `connections-speko-native-conversation--credits-required`, `connections-speko-native-conversation--idle`, `connections-speko-native-conversation--delayed-result`, `connections-speko-native-conversation--notification-unavailable`, `connections-speko-native-conversation--muted`, `connections-speko-native-conversation--ended`, `connections-speko-native-conversation--creation-recovery`, `connections-speko-native-conversation--pending-hangup`, `connections-speko-native-conversation--interrupted-hint-recovery`, `connections-speko-native-conversation--continue-conversation`, `connections-speko-native-conversation--new-conversation`, `connections-speko-native-conversation--access-revoked`, `connections-speko-native-conversation--reload-recovery`, `connections-speko-native-conversation--pause-within-spoken-answer` | +| `./storybook/stories/chat-channels/speko/speko-pages.stories.tsx` | `connections-speko-pages--catalog`, `connections-speko-pages--choose-agent`, `connections-speko-pages--choose-agent-long-name`, `connections-speko-pages--choose-agent-empty`, `connections-speko-pages--configure-provider`, `connections-speko-pages--test-conversation`, `connections-speko-pages--active`, `connections-speko-pages--paused`, `connections-speko-pages--attention`, `connections-speko-pages--revoked`, `connections-speko-pages--removed`, `connections-speko-pages--loading-failure`, `connections-speko-pages--private-access`, `connections-speko-pages--conversations`, `connections-speko-pages--conversations-empty`, `connections-speko-pages--activity-empty`, `connections-speko-pages--identity-provider-label`, `connections-speko-pages--setup-keeps-resume-link`, `connections-speko-pages--pending-incoming`, `connections-speko-pages--guest-intake`, `connections-speko-pages--with-call-history`, `connections-speko-pages--settings-without-call-history`, `connections-speko-pages--activity-with-mixed-entries`, `connections-speko-pages--activity-unapproved-call`, `connections-speko-pages--activity-call-history-failure`, `connections-speko-pages--paused-call-history`, `connections-speko-pages--long-agent-header` | +| `./storybook/stories/chat-channels/speko/speko-phone-callback.stories.tsx` | `connections-speko-phone-callback--initial`, `connections-speko-phone-callback--loading`, `connections-speko-phone-callback--enabled`, `connections-speko-phone-callback--disabled`, `connections-speko-phone-callback--saving`, `connections-speko-phone-callback--saved`, `connections-speko-phone-callback--provider-failure`, `connections-speko-phone-callback--validation-and-keyboard-save` | +| `./storybook/stories/chat-channels/speko/speko-provider-setup.stories.tsx` | `connections-speko-provider-setup--initial`, `connections-speko-provider-setup--credentials-entered`, `connections-speko-provider-setup--missing-https`, `connections-speko-provider-setup--verification-loading`, `connections-speko-provider-setup--resumable-setup`, `connections-speko-provider-setup--long-agent-name`, `connections-speko-provider-setup--keyboard-validation-and-connect`, `connections-speko-provider-setup--interrupted-before-credentials` | +| `./storybook/stories/chat-channels/speko/speko-task-launcher.stories.tsx` | `connections-speko-task-launcher--existing-agent`, `connections-speko-task-launcher--multiple-personas`, `connections-speko-task-launcher--unavailable-agent`, `connections-speko-task-launcher--agent-channels`, `connections-speko-task-launcher--connection-lifecycle`, `connections-speko-task-launcher--bound-task-banner`, `connections-speko-task-launcher--keyboard-open`, `connections-speko-task-launcher--connection-load-failure`, `connections-speko-task-launcher--bound-connection-recovery`, `connections-speko-task-launcher--persona-after-panel-close`, `connections-speko-task-launcher--shared-pause-resume` | +| `./storybook/stories/chat-channels/speko/call-history.stories.tsx` | `connections-speko-call-history--empty`, `connections-speko-call-history--loading`, `connections-speko-call-history--failure`, `connections-speko-call-history--completed`, `connections-speko-call-history--active`, `connections-speko-call-history--failed`, `connections-speko-call-history--missed`, `connections-speko-call-history--transcript-unavailable`, `connections-speko-call-history--partial-cost`, `connections-speko-call-history--interrupted-answer`, `connections-speko-call-history--long-conversation`, `connections-speko-call-history--keyboard-disclosure`, `connections-speko-call-history--end-call-focus`, `connections-speko-call-history--unapproved-incoming`, `connections-speko-call-history--revoked-history`, `connections-speko-call-history--activity-entry`, `connections-speko-call-history--activity-unapproved-entry`, `connections-speko-call-history--activity-end-call-focus`, `connections-speko-call-history--incoming-failure-preserves-history`, `connections-speko-call-history--end-failure-preserves-history` | +| `./storybook/stories/chat-channels/speko/incoming-call.stories.tsx` | `connections-speko-live-phone-approval--awaiting-approval`, `connections-speko-live-phone-approval--existing-task`, `connections-speko-live-phone-approval--confirming`, `connections-speko-live-phone-approval--approved`, `connections-speko-live-phone-approval--denied`, `connections-speko-live-phone-approval--expired`, `connections-speko-live-phone-approval--hung-up`, `connections-speko-live-phone-approval--failure`, `connections-speko-live-phone-approval--keyboard-approval-and-focus`, `connections-speko-live-phone-approval--guest-intake`, `connections-speko-live-phone-approval--public-keyboard-end`, `connections-speko-live-phone-approval--task-list-failure-and-recovery`, `connections-speko-live-phone-approval--approval-confirmation-expires` | +| `./storybook/stories/chat-channels/speko/phone-line.stories.tsx` | `connections-speko-incoming-phone-line--initial`, `connections-speko-incoming-phone-line--loading`, `connections-speko-incoming-phone-line--empty-inventory`, `connections-speko-incoming-phone-line--selected`, `connections-speko-incoming-phone-line--verification-required`, `connections-speko-incoming-phone-line--insufficient-credits`, `connections-speko-incoming-phone-line--provider-failure`, `connections-speko-incoming-phone-line--long-labels`, `connections-speko-incoming-phone-line--keyboard-save`, `connections-speko-incoming-phone-line--guest-intake`, `connections-speko-incoming-phone-line--private-task-approval`, `connections-speko-incoming-phone-line--selected-sandbox` | +| `./storybook/stories/chat-channels/speko/voice-controls.stories.tsx` | `connections-speko-voice-controls--idle`, `connections-speko-voice-controls--connecting`, `connections-speko-voice-controls--listening`, `connections-speko-voice-controls--speaking`, `connections-speko-voice-controls--muted`, `connections-speko-voice-controls--working`, `connections-speko-voice-controls--interrupted`, `connections-speko-voice-controls--reconnecting`, `connections-speko-voice-controls--ending`, `connections-speko-voice-controls--cleanup-failed`, `connections-speko-voice-controls--ended`, `connections-speko-voice-controls--failed`, `connections-speko-voice-controls--microphone-denied`, `connections-speko-voice-controls--playback-blocked`, `connections-speko-voice-controls--long-name`, `connections-speko-voice-controls--state-matrix`, `connections-speko-voice-controls--keyboard-mute-and-end`, `connections-speko-voice-controls--repeat-answer`, `connections-speko-voice-controls--repeat-unavailable-while-ending` | +| `./storybook/stories/chat-channels/speko/voice-conversation-panel.stories.tsx` | `connections-speko-conversation--idle`, `connections-speko-conversation--connected`, `connections-speko-conversation--connection-recovery`, `connections-speko-conversation--cleanup-recovery` | +| `./storybook/stories/chat-channels/speko/voice-transcript.stories.tsx` | `connections-speko-transcript--empty`, `connections-speko-transcript--unavailable`, `connections-speko-transcript--partial-speech`, `connections-speko-transcript--final-speech`, `connections-speko-transcript--overlapping-updates`, `connections-speko-transcript--delayed-result`, `connections-speko-transcript--task-reference`, `connections-speko-transcript--long-conversation` | + + +## Production component mapping + +| Component | Story module(s) | +| --- | --- | +| `VoiceControls` | `voice-controls.stories.tsx` | +| `VoiceTranscript` | `voice-transcript.stories.tsx` | +| `VoiceConversationPanel` | `voice-conversation-panel.stories.tsx`, native conversation stories | +| `NativeVoiceConversation`, `VoiceSessionView`, `ConversationChoice` | `speko-native-conversation.stories.tsx` | +| `VoiceCallProvider`, application call host | `speko-call-provider.stories.tsx` (StrictMode, route navigation, company/logout cleanup, panel restart, focus restoration and a bounded long transcript) | +| Changed `StatusBadge` behavior | `speko-connection-status.stories.tsx` and full-page states | +| `SpekoPhoneCallbackForm`, `SpekoPhoneCallback` | `speko-phone-callback.stories.tsx`, full connection Settings page; keyboard opt-in/save, invalid input, retryable errors, fictional phone only | +| `SpekoCallHistoryView`, `SpekoCallReport`, `SpekoCallActivityItem`, `SpekoUnapprovedCallActivityItem` | `speko/call-history.stories.tsx`, full Activity page; safe costs, interrupted/long/missing transcripts, current-access revocation, keyboard disclosure, call end/focus restoration, unapproved calls, and retained transcripts after inbound-history/end-call errors | +| `SpekoPhoneLine`, `SpekoPhoneLineForm` | `speko/phone-line.stories.tsx`; loading/empty inventory, verification/credits/provider failure, private/guest line setup and keyboard save | +| `SpekoIncomingCalls`, `SpekoIncomingCallApproval` | `speko/incoming-call.stories.tsx`; pending/approve/deny/expired/guest/hangup, long task names, validation and recovery | +| `SpekoProviderSetup` | `speko-provider-setup.stories.tsx` | +| `TaskVoiceLauncher`, `AgentChannelsPanel`, `ExternallyConnectedTaskBanner` | `speko-task-launcher.stories.tsx` | +| `Browse`, `ChatEndpointSetup`, `ChatEndpointDetail`, `ChatIdentityConfirm` | `speko-pages.stories.tsx` | + +## Checks and outstanding coverage + +- Interaction checks include validation, mute/end/retry, paced notification recovery, partial speech with pauses, explicit repeat, reload cleanup, Continue/New conversation, revoked access, route navigation, keyboard launch, and focus restoration after template/call dismissal. +- Accessibility checks include dialog portals, contrast, accessible names, overflow, and final-only live transcript announcements. Fixed template focus restoration, manager-role/badge contrast, and narrow persona selection. +- The existing visual suite discovers Speko stories. October screenshots and interaction reports are in `tests/storybook-visual/test-results/` and `tests/storybook-visual/playwright-report/index.html`. The October run checks interactions, accessibility, external-network blocking and overflow, and captures screenshots; the previous local macOS comparison passed 274 snapshots before the four final template scenarios. At the October 7 checkpoint, all 284 Speko story/theme comparisons passed: 282 existing comparisons in `speko-completion-final-visual.log` and two newly inspected credit-error candidates in `speko-completion-credits-reviewed-visual.log`. The first invocation recorded missing baselines for those two new candidates; both were visually inspected before the successful comparison. Reviewed the empty-manager template, private approval, completed/history, narrow and dark fixtures. Linux CI baseline approval/publication remains outstanding. Reviewed narrow light call-host, long-conversation, and dark repeat-control and in-panel restart screenshots. September's 140 local baseline comparisons remain historical; new/changed baseline approval and CI publication are outstanding. +- Design Guide includes voice controls, transcript, the production Speko setup form, callback form, phone line, incoming-call approval and call history. The task launcher has dedicated stories; the installer was removed from Speko setup on October 8. +- Full Apps Browse/Setup/Detail/Identity stories cover setup steps, active/paused/attention/revoked/removed connections, private access guidance, load failure, conversation history and empty activity. The removed installer’s failure/loading/manager stories were covered at the October 7 checkpoint. Real-template installation has historical live UI proof; catalog-install retry was covered by the October 7 E02 test; October 8 E02 verifies existing-agent selection and preserved configuration instead; arbitrary concurrent catalog-import idempotency and normal-motion/hardware acceptance remain unqualified. +- Personal outgoing callback configuration is implemented. Provider-number inventory, phone approval/guest intake, session history and costs have the component stories listed above. Separate per-voice catalog selection remains outside the persona-based setup. + +Commands: `pnpm build-storybook`, `pnpm check:token-gates`, and `STORYBOOK_VISUAL_PORT=6119 pnpm exec playwright test --config tests/storybook-visual/playwright.config.ts tests/storybook-visual/speko-voice.spec.ts`. The optional port override avoids accidentally reusing another worktree's Storybook server. Do not blanket-update unrelated baselines. + +October 7 final retry coverage: the production `TeamInstallerDialog` accepts a default collision strategy; the removed Speko installer supplied `skip` and preserved an installed matching agent on retry. `CreationFailure` checks both actual preview/install options. All eight template stories passed 32 interaction/accessibility checks and 16 unchanged visual comparisons after this correction. The real-server E02 journey also preserves the original agent configuration and historical task assignment. + +October 7 credit-failure coverage: `CreditsRequired` uses the real production conversation and deterministic HTTP 409 fixture. Four theme/width interaction/accessibility checks and two visually inspected theme comparisons passed. Real-server E12 proves funding guidance, exact failed-session cleanup, redacted browser response, durable provider rejection, and a replay with only one external creation attempt. No actual microphone or provider call occurs in these fixtures. + +October 8 onboarding change: removed the dedicated template component and its eight installer stories. The optional receptionist remains in the team catalog, independent of connection setup. Speko instructions now use the same automatic, assignment-scoped connector skill delivery as AgentMail; task communication guidance also reaches the selected voice agent. The existing `TeamInstallerDialog` retains its generic catalog stories. + +October 8 visuals: six intentional agent-selection light/dark baselines were reviewed and updated individually; all six subsequent comparisons passed (`/tmp/speko-existing-agent-visual-comparison.log`). No unrelated snapshots were updated. + +October 8 backend and live-app verification: 65 focused guidance/session tests and all three real-server integrated Speko journeys passed, including existing-agent selection/reload with unchanged runtime and historical assignment. The runtime task snapshot contains the voice guidance; automatic connector skill coverage verifies supported adapter delivery, company/agent isolation, disabled callbacks, pause/revoke/remove, disabled credentials, removed assignment and the experimental gate. UI/server typechecks, UI/Storybook/catalog builds and token gates passed. The running localhost:3469 app was reloaded and checked in the embedded browser. No provider call or microphone was used for this change. + +October 8 shared-path regressions: 65 AgentMail/Browser Use connector tests and 11 catalog tests passed. The shipped-agent catalog expectation was brought into sync with the already present optional Speko agent. These targeted checks qualify this onboarding/instruction change; they do not supersede the outstanding whole-repository and real-provider acceptance work in the October 7 qualification report. + + +October 8 call Activity change: removed Your calls from connection Settings. Browser, incoming and outgoing sessions plus unapproved calls now appear chronologically alongside connection events on the first Activity page, using the same caller-authorized history APIs. Expanding a call retains its task link, transcript, provider charge and authorized end-call control. Connection-event pagination remains unchanged; recent caller-visible calls appear only once on the first page. The two extracted entry components have dedicated production-component stories and Design Guide examples. + +The changed page/call-component subset passed 176 interaction/accessibility checks (44 stories × two themes × two widths), including chronological order, absence from Settings, transcript/task/cost disclosure, unapproved calls, history denial, paused history and end-call focus restoration. All three real-server integrated journeys passed; E15 now checks call reports in Activity and absence from Settings. UI typecheck, UI/Storybook builds and token gates passed. The localhost:3469 app was reloaded and verified in the embedded browser without placing a call or requesting a microphone. Test logs use `/tmp/speko-call-activity-*.log`. + +Visual review inspected the 26 intentional/new light/dark candidates: ten changed existing snapshots and sixteen new ones. The first broad invocation used a missing default baseline directory; comparisons were then repeated against the existing reviewed `/tmp/speko-visual-candidates` directory. All 62 unchanged snapshots matched before the ten intentional changes were individually accepted. All 88 final comparisons passed (`/tmp/speko-call-activity-visual-final.log`); Linux baseline publication remains separate from this local review. This UI qualification does not replace outstanding real-provider or whole-repository acceptance. + + +October 8 Settings simplification: removed the browser conversation selector/launcher and its introductory section from connection Settings. Setup's optional voice test and task-based voice launcher remain. The shared Speko connection header now uses the bundled AppLogo and an Open Speko account link. Callback configuration no longer includes the two introductory paragraphs. Incoming-call and Access copy explain the existing pairing flow: a private call reads a code that must be entered in signed-in Paperclip for that live call and a permitted task. Guest intake takes messages for review without private task access or agent execution. No authentication or provider contract changed. SettingsWithoutCallHistory covers absence of both history and browser launchers, the account link and bundled logo; LongAgentHeader covers wrapping. Callback and phone-line component stories cover their shorter copy. + +The Conversations empty state now directs users to Talk to agent on an assigned task. The Settings cleanup passed three real-server integrated journeys, 180 page/form/task-launcher interaction/accessibility checks plus four long-header checks, UI typecheck/build and token gates. Test evidence uses `/tmp/speko-settings-cleanup-*.log`; microphone/provider calls were not made for this change. Visual review covers the changed header and shorter forms in both themes; broader live-provider and repository-wide acceptance remains separate. + +Final Settings verification: four additional interaction/accessibility checks passed for the corrected Conversations empty state. Reviewed 70 changed candidates and both new long-header theme snapshots; after correcting the stale empty-state instruction, all 90 final affected visual comparisons passed against the reviewed local baseline directory (`/tmp/speko-settings-cleanup-visual-final.log`). No unrelated snapshots were updated. The final UI/Storybook builds passed; the dev server was reloaded and the saved connection Settings verified in the embedded browser. Linux baseline publication and whole-feature qualification remain separate. + +## October 8 low-trust incoming conversations + +The historical message-only guest-intake descriptions above are superseded for newly admitted opted-in calls. `SpekoPhoneLine` now offers a connection-scoped sandbox choice, `SpekoIncomingCalls` shows a conversation task and End call without a code, and the changed Access page explains both public and private modes. The existing `guest-intake` story IDs are retained for compatibility and now render the actual public conversation components. PrivateTaskApproval, SelectedSandbox and PublicKeyboardEnd add explicit private/default, sandbox selection and keyboard/focus coverage. No new agent is installed. + +Historical development run: all 50 affected production-component stories passed 200 interaction/accessibility checks across light/dark themes and desktop/narrow viewports with reduced motion. Reviewed all 40 intentional/new visual candidates individually, accepted only those exact story IDs, and passed all 100 final affected theme comparisons. The built inventory has 148 Speko stories. UI and Storybook builds, UI typecheck and token gates passed. Local macOS baseline review does not publish canonical Linux baselines or qualify telephone audio. See [incoming-call qualification](2026-10-09-speko-pr-qualification.md). + +Final merge preparation adds a bounded approval-confirmation story using the production incoming-call component. Its play function approves the call, verifies focus, and waits for the five-second receipt to clear. The prior 220-check result remains historical; all 108 final affected approval/conversation checks passed and are recorded in the qualification report. + +Latest interrupted-setup repair: the 155-story build and all 32 provider-setup interaction/accessibility checks passed across both themes and 390/1200-pixel widths with reduced motion. The production story verifies generation of a missing signing secret during interrupted setup; existing reconnect stories preserve saved secrets. No real microphone or external provider requests were used. + +October 13 review repair: the task launcher restores persona selection and keyboard focus after the hosted voice panel closes. PersonaAfterPanelClose exercises the production launcher without opening media. OverlappingUpdates verifies that interrupted transcript fragments are labeled Interrupted and no longer Speaking. The Team Catalog data helper is explicitly excluded from the Storybook index. + +October 13 verification: the 156-story build and all 76 affected launcher/transcript interaction and accessibility checks passed across both themes and 390/1200 widths with reduced motion. Narrow-screen captures were inspected. After master adopted generated indexes, Speko source annotations retained stable story IDs and explicit fixture/provider limits; all 28 catalog/map structural tests passed. Structural checks do not establish runtime or provider acceptance. + +The complete generated-catalog Storybook run passed all 624 checks for the 156 Speko stories (UI source 249cbc616a). These are fixture interaction/accessibility results and viewport captures, not provider or spoken-playback acceptance. + +The shared endpoint-cache repair adds SharedPauseResume to the source inventory. The mounted production launcher observes the same company-scoped cache invalidated by setup and lifecycle controls. All 27 focused UI regressions, UI typecheck and token gates pass; its affected Storybook verification is recorded in the PR after execution. diff --git a/doc/plans/2026-09-11-speko-voice-e2e-runbook.md b/doc/plans/2026-09-11-speko-voice-e2e-runbook.md new file mode 100644 index 00000000000..2e59578412b --- /dev/null +++ b/doc/plans/2026-09-11-speko-voice-e2e-runbook.md @@ -0,0 +1,440 @@ +# Speko voice connection: implementation and E2E runbook + +Date: 2026-09-11; updated 2026-09-12. Status: native browser milestone proven; +telephone and full E01–E16 qualification outstanding. See the +[native qualification report](2026-10-09-speko-pr-qualification.md). +Branch: `codex/speko-chat-connection`. +Base: `d10cbde81523a13344fc18a1f70a76edac8a93fc` (clean current master). + +## October 9 live-call push + +Telephone replies now use `POST /v1/calls/{id}/messages` with `mode: "respond"`. +Keep the call open after submitting work and verify automatic output without +`get_updates` polling or a result-ready hint. Check the durable action receipt +and task reply separately from confirmed spoken playback. A 409 ends delivery; +uncertain requests must not be automatically resent. Browser result hints are +unchanged. See the [live-call push qualification](2026-10-09-speko-pr-qualification.md) +for the real-provider text-input proof, timings, reproduction and remaining +PSTN/interruption qualification. + +## October task-parity qualification + +Updated 2026-10-07: September's browser proof is historical. The current task-parity +implementation still requires live audio requalification; see the +[October qualification report](2026-10-09-speko-pr-qualification.md). + +Run deterministic integration without a Speko account, microphone or public tunnel: + +```sh +npx --yes pnpm@9.15.4 --filter @paperclipai/ui build +npx --yes pnpm@9.15.4 exec tsc -p tests/speko-native/tsconfig.json +npx --yes pnpm@9.15.4 exec playwright test --config tests/speko-native/playwright.config.ts +npx --yes pnpm@9.15.4 exec playwright test --config tests/speko-native/playwright-auth.config.ts +``` + +The first suite uses loopback ports 3480/3481 and a fresh local-trusted instance. +It covers delayed work, deduplication, route navigation, repeat, reload cleanup, +explicit new conversations, end/rejoin, results completed after hangup, and typed +questions answered through canonical task interactions. The fixture supervisor +retains the database across app restarts; external fixture state survives separately. +Only deterministic stub mode accepts the local restart signal. The test verifies +that an open call survives an orderly app restart, its repeated answer callback +uses the original receipt, and explicit Repeat retrieves the saved response +without creating sessions, comments or runs. The +second uses 3482/3483, real Better Auth signup and invitation APIs, and separate +operator/member/outsider/anonymous contexts. It checks two-company isolation, +private tasks, caller ownership and mid-call membership revocation. Only Speko +transport/media and execution are stubbed. No Paperclip API response is mocked. + +Each suite shuts down its own server/database. Its generated HTML report stays +under `tests/speko-native/playwright-report/` or `playwright-report-auth/`. +The integration report includes synthetic durable records and a screenshot; +authentication traces are off so session cookies/passwords are not captured. +Do not count these fixture runs as live audio or full E01–E16 acceptance. + +## Native browser acceptance commands + +Use pnpm 9.15.4 to honor this checkout's dependency patches. Build the static UI +before starting the temporary server: + +```sh +npx --yes pnpm@9.15.4 --filter @paperclipai/ui build +SPEKO_NATIVE_PORT=3449 SPEKO_NATIVE_CALLBACK_ORIGIN=https://your-test-origin.example \ + server/node_modules/.bin/tsx tests/speko-native/server.ts +``` + +The launcher creates a fresh OS temporary home and embedded PostgreSQL database, +removes inherited database/worktree configuration, and binds the app to loopback. +The parent owns the database; the child watches app code for restart tests. Expose +only `/api/voice-webhooks` through the explicitly configured HTTPS origin. Preserve +other services on any shared proxy. Do not run builds or edit server imports +during a live call. The static UI avoids Vite reloads interrupting media tests. + +In the temporary UI, create an acceptance company and existing agent. The harness +substitutes only the `process` execution adapter with a 61-second fixture. Other +adapters, including the catalog template's `claude_local`, remain real. Install +the optional receptionist template through the team catalog when testing that path; Apps setup selects an existing agent. Give it an +isolated working directory with an `ACCEPTANCE.txt` file containing the synthetic +word `marigold`; do not point acceptance work at private project files. + +Create a dedicated hosted Speko persona per endpoint. Its signed tools are managed +by the actual setup UI. Save a private fixture at +`~/.paperclip/speko-proof/native-fixture.json` (existing agent) or +`native-template-fixture.json` (template), mode 0600: + +```json +{ + "companyId": "temporary-company-uuid", + "companyPrefix": "SPE", + "agentId": "selected-agent-uuid", + "agentName": "Existing Paperclip Agent", + "providerAgentId": "agent_dedicated_persona", + "endpointId": "optional-resumable-endpoint-uuid" +} +``` + +Omit `endpointId` for new setup. The live harness fills the real masked setup form +using `SPEKO_MCP_API_KEY` from `~/.secrets` and persists only the endpoint ID in the +fixture. Do not record Playwright traces around credential entry. + +Prepare synthetic input using `say` and `ffmpeg`, saving mono 48 kHz signed 16-bit +little-endian PCM files under `.paperclip-local/speko-proof-audio/`: + +- `start.pcm`: “Start the test.” +- `followup.pcm`: “Add a follow-up.” +- `template-start.pcm`: “Read ACCEPTANCE dot T X T in your working directory and tell me the verification word.” +- `template-delayed-start.pcm`: “Run python three voice underscore acceptance underscore benchmark dot py in your working directory. It computes for sixty five seconds. Wait for the command to complete, then tell me the verification word from its output.” +- `template-delayed-followup.pcm`: “Add another instruction to that task. Include the phrase silver lantern in the final answer.” +- `template-followup.pcm`: “Please include the verification word in your answer.” +- `template-resume.pcm`: “Please tell me the verification word from earlier.” + +```sh +node tests/speko-native/live-browser.mjs +SPEKO_NATIVE_SCENARIO=template node tests/speko-native/live-browser.mjs +SPEKO_NATIVE_SCENARIO=template-delayed SPEKO_NATIVE_EXPECTED_WORD=cobalt \ + SPEKO_NATIVE_EVIDENCE_PATH=/your/temporary/home/evidence.json \ + node tests/speko-native/live-browser.mjs +SPEKO_NATIVE_SCENARIO=template SPEKO_NATIVE_RESUME_ISSUE_ID=existing-task-uuid \ + node tests/speko-native/live-browser.mjs +``` + +For the delayed real-agent case, copy `tests/speko-native/fixtures/acceptance-benchmark.py` to `voice_acceptance_benchmark.py` in the isolated agent workspace. It performs a 65-second SHA-256 computation on one CPU core, then reads `ACCEPTANCE.txt`; it has no network access. Change the isolated file to the expected word before the call and allow at least 180 seconds in the selected agent runtime. The harness requires two accepted session tool receipts plus an agent-authored task comment containing the distinct follow-up phrase. Confirm the benchmark completed its computation and independently check the received audio; a transcribed follow-up alone does not prove delivery. + +The script uses the real UI, server, database, queues and provider. It replaces +only microphone input, records received audio privately, and asserts live +transcript, task binding, mute state, terminal session state and stopped tracks. +Independently transcribe or listen to the received audio before declaring spoken +success. Confirm the exact provider call is ended. A matching word elsewhere in +the task page is not proof. `SPEKO_NATIVE_IDLE_PROBE=1` deliberately disables the +browser notification feed for provider capability experiments and is never a +normal integration pass. + +Inspect `instance.json`, `evidence.json` and `execution.jsonl` in the temporary +home for durable session/delivery/publication/task evidence. Each browser run +writes a private source manifest, report, screenshots and received audio under +`~/.paperclip/speko-proof/native-browser-*`. The checked-in reports contain only +synthetic results and numeric references. On cleanup, end active provider calls, +stop only this launcher, and remove only this test's proxy path. + +Targeted backend verification: + +```sh +npx --yes pnpm@9.15.4 exec vitest run \ + server/src/__tests__/voice-session-store.integration.test.ts \ + server/src/__tests__/speko-protocol.test.ts \ + server/src/__tests__/speko-provider.test.ts \ + packages/db/src/speko-voice-migration.test.ts \ + ui/src/lib/voice-session-controller.test.ts +``` + +This harness is a live-provider qualification tool, not the complete deterministic +two-company E01–E16 browser suite described below. + +## Product contract and gate + +Build a native experimental Apps voice connection beside Slack/Discord. Speko +owns hosted audio, conversational acknowledgment, and telephony; the selected +Paperclip agent keeps its runtime, tools, task context, and work. Support browser +voice, inbound telephone, and authenticated “Call my phone.” Recommend the +optional **Speko Company Phone Agent** catalog template, while permitting any +existing agent. Do not automatically install the template in every company. + +Before production changes, prove a hosted live conversation can wait for work +lasting at least 60 seconds, handle an interruption and additional instruction, +then automatically speak the eventual result. Tool requests have a four-second +ceiling; use quick submission and cursor retrieval with bounded waits. Browser +context injection is not a working alternative in the documented client. If the +gate fails, record the missing capability and founder questions. Do not replace +this product with after-call processing or a self-hosted voice stack. + +Sources checked 2026-09-11: + +- [Hosted tool calling, timeout and signing](https://docs.speko.ai/guides/tool-calling) +- [Browser data-channel limitations](https://docs.speko.ai/client/data-channel) +- [Browser lifecycle](https://docs.speko.ai/client/voice-conversation) +- [Hosted session credentials and duration limits](https://docs.speko.ai/api-reference/sessions) +- [Telephony and number requirements](https://docs.speko.ai/guides/phone-agents) +- [Lifecycle webhook delivery behavior](https://docs.speko.ai/guides/webhooks) +- [Original catalog PR #11418](https://github.com/paperclipai/paperclip/pull/11418) + +PR #11418 describes the general-purpose Speko MCP catalog connection; it is +research input, not the live voice runtime. Keep general MCP tools separate. + +## Implementation after gate success + +1. Add explicit voice runtime/transport to the shared app/channel contracts and + validators, database constraints, server lifecycle and UI API clients. Keep + `enableChatConnectors` as the discovery/setup gate. +2. Reuse company endpoints, identities, task bindings, durable deliveries, + external publications and activity logging. Vault API/signing secrets. Each + endpoint binds one Speko persona and one Paperclip agent; preserve historical + assignments when configuration changes. +3. Persist session company, endpoint, provider ID, caller authority, task, + generation, lifecycle, expiry and reply cursor. Authenticated APIs create, + inspect, approve/deny private inbound sessions and end calls. Signed callbacks + verify raw body, time, session identity and idempotency before side effects. +4. Task entry resumes authorized work; agent entry creates one task. Follow-ups + use existing steering/queue behavior. Playback interruption, work cancellation + and call ending are separate transitions. Expose only permitted answers, + questions and safe progress. Distinguish result delivery from spoken evidence. +5. Setup: credentials → select/create agent → voice → number → test. Offer + existing Speko inventory, and guide purchase/import through Speko. Present + actionable verification, credits, HTTPS and device errors. Add task voice + controls, transcript, incoming-call approval, history and connection management. +6. Package the recommended agent in the existing template catalog: greeting, + intake, clarification, delegation and concise follow-up. Creation follows + normal company permissions and is idempotent. + +Browser authority inherits the signed-in user's current access; only short-lived +media credentials enter the browser. Inbound caller ID grants no private access. +Private calls require authenticated approval of the specific live call. Guest +intake is explicitly enabled and cannot browse private tasks or approve governed +actions. Recheck current authorization throughout a session, including after a +company switch, revocation, pause or removal. + +Callbacks require reachable HTTPS; local instances need a configured tunnel or +reverse proxy. No private relay is included. Retain transcripts/task history, +explain provider recording behavior and do not download recordings by default. +Set session-duration limits separately from join-token TTL and display Speko +charges separately from Paperclip agent costs. Exclude SMS, WhatsApp, campaigns, +autonomous outbound calls and general telephony administration. + +## A. Deterministic UI and integrated browser qualification + +Extend the existing chat UI suite for fixtures. Add a separate integration suite +with a real Paperclip server, isolated database, permissions, queues and external +publication path. Stub **only** Speko's transport and the execution provider. +Use temporary instances, two companies, operator/member/guest contexts, the +template agent and an existing agent. Do not reuse the existing UI suite's +mocked Paperclip responses as evidence that the backend works. + +Every row must assert visible UI, resulting task state and durable session and +delivery records. Wait on observable transitions and receipts, not fixed sleeps. + +| ID | Journey | Required evidence | +| --- | --- | --- | +| E01 | Connect through Apps | Secret masked/no browser durable key; draft persists on reload; activation requires verification | +| E02 | Recommended or existing agent | Correct assignment; creation permission; failed/retried creation never duplicates agents | +| E03 | Browser voice from task | Authorized task/session binding; short token expiry; provider key never reaches browser or trace | +| E04 | Agent entry and later resume | Exactly one task on initial admission; authorized later call resumes context without reassignment | +| E05 | Work lasts 60+ seconds | Prompt acknowledgment; safe progress; correct automatic final answer; one task and accepted request | +| E06 | Interrupt and follow up | Playback stops; ordered follow-ups reach same task; work continues unless explicitly cancelled | +| E07 | Cancel versus hang up | Independent work/call lifecycles; history and running work survive hangup | +| E08 | Private inbound | No private disclosure before approval; approve/deny/expiry; spoofed caller ID confers no authority | +| E09 | Company guest intake | Restricted intake succeeds; private task lookup and governed actions fail | +| E10 | Call my phone | Authenticated caller/task; duplicate click does not redial; ambiguous provider result reconciles first | +| E11 | Questions and approvals | Permitted question answers steer work; governed approvals retain existing checks | +| E12 | Provider/network failure | Timeouts, rate limits, credential expiry, disconnect and reload yield actionable recoverable state | +| E13 | Restart/retry | Duplicate and out-of-order callbacks plus process restart never duplicate tasks/comments or replay stale speech | +| E14 | Permissions/lifecycle | Revocation, pause, reconnect, removal and company switch enforce current access mid-call | +| E15 | History/reports | Transcript/task references agree; repeated report updates one record; no recording download | +| E16 | Media/responsiveness | Denied/missing mic, mute, navigation, unmount and end clean up media; usable narrow layout | + +## B. Protocol, security and persistence + +Test valid/invalid signatures, exact raw-body tampering, stale/future timestamps, +replay, secret rotation overlap, cross-company/endpoint/session substitutions, +stale generations, closed sessions, cursor bounds and replay of already spoken +publications. Verify scoped admission and current authority again at delivery. +Probe event redaction for keys, raw agent logs, tool outputs, private comments and +unapproved answers. Assert delivery and playback are separate durable facts. + +Simulate concurrent delivery, uncertain outbound acceptance, reordered reports, +missing lifecycle callbacks and restart at each external-side-effect boundary. +Verify additive migrations against both an empty database and an upgraded copy; +check constraints and backfill defaults for existing Slack/Discord connections. + +## C. Real-provider acceptance + +Use a dedicated Speko workspace, durable reachable HTTPS callback origin and +designated phone numbers. Record exact tested commit and configuration. Complete +setup with both agent choices, then test real browser, inbound and outbound calls. +Repeat delayed work with interruption and follow-up on both browser and phone. +Demonstrate private-call approval and guest intake, hangup/reconnect/resume, +credential rotation, callback outage/recovery and revocation. Include desktop, +mobile and Safari microphone/playback behavior. + +Listen to output: intelligibility, silence, turn-taking, duplicate speech, +interruption and answer accuracy. Record speech-end→acknowledgment (target ≤2s), +result-available→audio (≤5s), and interrupt→playback-stop (≤500ms) under healthy +conditions. Report actual measurements, clock sources and execution time +separately. Capture test audio only with participant consent. Keep keys/private +transcripts out of generic traces and artifacts. + +Statuses are **passed**, **failed**, **blocked**, or **not run**. Fixture success +does not substitute for real audio/telephone proof. Keep Speko experimental until +all three transports and delayed-result behavior pass. + +## UI/Storybook and completion gates + +Use the companion component-to-story inventory. All stories render production +components with deterministic fixtures and no real microphone/provider traffic. +Add changed shared behavior, keyboard/focus/validation/control/recovery tests, +accessibility checks and design-guide examples. Run token gates and Storybook +build. Include stories in the existing Playwright visual suite, review intentional +baselines on its supported environment, and never blanket-update unrelated ones. + +Before PR-ready handoff run targeted checks, integrated Speko E2E, existing +chat-provider regression, `pnpm -r typecheck`, `pnpm test:run`, `pnpm build`, +Storybook interactions/accessibility and visual tests. Report anything blocked. +Use the repository PR template if submitting a PR. + +Qualification records must include UTC date, tested SHA/dirty patch, case IDs, +fixture/live environment, roles/company boundaries, observed timings, sanitized +session/task references, screenshot/report paths, failures and reproduction steps. +The current first-gate report is `2026-09-11-speko-voice-qualification.md`. + + +## 2026-10-07 authorized tunnel and playback retest + +The user explicitly authorized the Cloudflare test tunnel. The approved proxy +forwards only POST `/api/voice-webhooks/:publicId/tools`; board/UI routes and +all other APIs return 404. Provider signatures, session capabilities and current +company/task authority remain enforced. Quick-tunnel URLs are temporary; +verify the origin before each run and retain no promise of permanent availability. + +Audit received audio independently of the UI transcript. A passing live-browser +script is provisional until the whole result and follow-up have been heard or +independently transcribed from the actual received stream. An SDK `listening` +event can occur between words; a final transcript is not confirmed playback. +The second October retest passed its script but failed this audio review. + +Retrieval now batches up to eight approved publications in cursor order, stopping +at an interaction boundary. This avoids a separate typed turn for each already +queued result. Streaming publications, current-access checks, repeat semantics, +idempotent receipts and unknown playback status remain unchanged. It does not +prove that a later asynchronous publication can never interrupt ongoing speech. +Retest that case and the three timing targets before telephone expansion. + +The optional template was installed through the actual UI and its real Claude +runtime completed a 65-second Python workload, accepted the spoken follow-up, +and spoke the verification word plus the requested phrase. This establishes one +real runtime journey; it does not qualify template retry/idempotency, governed +approvals, human audio, Safari or telephone journeys. + + +## Agent-requested callback testing — updated 2026-10-07 + +1. Configure and verify a dedicated Speko persona on the selected agent's native + Apps connection. Keep experimental chat connectors enabled and expose only + signed voice callbacks through explicitly configured reachable HTTPS. +2. In connection **Settings → Call my phone**, save your own E.164 number and + enable **Allow this agent to call me about my tasks**. Speko charges and + recording settings apply; Paperclip does not download recordings. +3. Create an ordinary task assigned to that agent with yourself as responsible + person, requesting one call using the assigned `call_my_phone` connection + tool. The tool accepts no destination parameter and returns no media token. + Confirm actual runtime discovery, a durable gateway invocation, one task and + one `outbound_phone` session before counting placement as passed. +4. Answer the real telephone call. Screening/mailbox, first agent audio and SIP + answered/bridged do not prove a human task conversation. Record them separately. + For an answered call, submit work lasting at least 60 seconds, interrupt and + add an instruction. Assert signed callbacks, durable ordered comments, + selected runtime execution and correct approved audio without repeated caller + prompting. Measure acknowledgment/result/interruption with the stated targets. +5. Hang up and verify provider/Paperclip reconciliation. Rejoin only on an + explicitly authorized fresh attempt; uncertain creation must not blindly + redial. Revoking personal opt-in/current access must stop private delivery. +6. Retain sanitized task/session references and status. Do not include personal + numbers, credentials or private call transcripts in general traces. Review + consent before any audio capture; never automatically download recordings. + +The integrated `tests/speko-native/integrated.spec.ts` E10 extension uses real +Paperclip routes, database, task queue, run identity and runtime MCP. Only external +Speko and execution are fixtures. It saves callback settings through the UI, +reloads, asks the execution fixture to discover/call its actual assigned tool, +rejects a simultaneous second request, verifies one outgoing durable/provider +session and output redaction, ends the call without cancelling work and disables +opt-in. It does not qualify real telephone audio or call-screening behavior. + +## October 7 completion-audit procedure + +Current private PSTN setup uses per-live-call approval in Paperclip; caller ID alone grants no access. A signed call creates one short-lived admission without a task. Approve its exact six-digit code while selecting an authorized task, or deny it; expiry and hangup close admission. Discard speech submitted before approval. Guest mode must be explicitly enabled and writes bounded low-trust intake without waking an agent, exposing private work or approving governed actions. Promote only the exact live call through authenticated Paperclip approval. + +Phone polling is enabled in signed phone pre-call hints, never by mutating shared persona defaults on each dial. Verify that a subsequent browser call never reads idle polling instructions aloud. Provider final-report timestamps may be camelCase and durations fractional; the safe projection excludes application control hints, keys, system turns and recording URLs. Check both new and existing history entries. + +Final component inventory: 141 stories and 564 theme/viewport interaction/accessibility checks. Run the existing visual suite with an external snapshot directory; review only Speko changes and publish Linux-approved baselines through the existing workflow. Local macOS candidate agreement is not CI baseline approval. + +See the dated qualification report for actual provider session/task references, received-media timings, passed/failed/blocked labels and remaining E01–E16 gaps. No fixture result qualifies human phone conversation or Safari media. Do not redial an uncertain call or substitute an internal Speko agent-to-agent route for PSTN proof. + +For E02, repeat a catalog import with `collisionStrategy: skip` after its first commit and assert one installed phone agent, the same ID/configuration, and unchanged historical task assignment. The Speko production wizard defaults to that policy; other catalog imports retain their existing defaults. Distinguish this committed-import recovery from concurrent general catalog transaction idempotency. + +If Speko returns HTTP 402 / `INSUFFICIENT_CREDITS`, mark the affected actual-provider case blocked, preserve the rejection without keys, and stop retries. Funding is an operator action; fixture runs and UI transcripts cannot replace the blocked independent audio check. + +October 7 deterministic E12: exhaust the external transport fixture's credits for one persona, start through the actual Apps UI, and require the funding message. Assert no media connection, redacted provider identity, one durable failed attempt and exactly one provider create after idempotency replay. End/cleanup may retire the local failed call without creating another provider session. Run `tests/speko-native/integrated.spec.ts` for the complete real-server journey plus E02 and E12; live HTTP 402 remains blocked until the workspace is funded. + + +## October 8 call Activity regression + +In the real-server E15 journey, first verify that connection Settings has no Your calls region. Open the connection's Activity tab and refresh its combined feed. Find the completed incoming-call entry, expand it, and verify the saved transcript and task link against durable session/report records. Repeated provider reports must still update one record and preserve the original task assignment. Configure guest intake from Settings, then return to Activity and assert both authorized and guest call entries. Private denied/expired/missed admissions disclose no connected task. + +The production feed uses existing caller-authorized history APIs and merges recent calls with connection events by creation time. Recent calls appear once on the first Activity page; older connection-event pages keep their existing cursor. History remains available on paused connections when current caller/task access permits it. History failures retain connection events and expose Retry calls. + +Run the real-server native suite and the 44 page/call-entry Storybook stories. Fixtures must never request a real microphone or contact Speko. Include both themes, 390/1200 viewports, keyboard disclosure and end-call focus restoration. The October 8 run passed all three integrated journeys and 176 story interaction/accessibility checks, plus UI typecheck/build, Storybook build and token gates. Review and compare only intentional snapshots against the existing baseline directory; the complete affected visual subset has 88 theme comparisons. Whole-feature live telephone/audio and canonical repository checks remain governed by the October qualification report. + + +## October 8 simplified Settings regression + +After activating a Speko connection, open Settings and verify the bundled provider logo and Open Speko link. There must be no browser Conversation selector, Start voice button, Your calls region or callback introduction. Save/reload outgoing phone settings and incoming number settings normally. Setup's optional voice test remains available, and assigned tasks retain their task voice launcher. E10 checks the simplified Settings surface before saving callback preferences; E08/E09 still exercise signed incoming approval and restricted guest intake. + +The current private inbound flow requires the caller to also use signed-in Paperclip: enter the code spoken during this live call, select a permitted task and approve. The pending private admission expires after two minutes. This is a browser-mediated pairing flow, not sign-in by phone. Guest intake can record a message as quarantined work for review; it does not run the agent or expose private task information. This UI change preserves those authorization semantics. + +The Conversations empty state now directs users to Talk to agent on an assigned task. The Settings cleanup passed three real-server integrated journeys, 180 page/form/task-launcher interaction/accessibility checks plus four long-header checks, UI typecheck/build and token gates. Test evidence uses `/tmp/speko-settings-cleanup-*.log`; microphone/provider calls were not made for this change. Visual review covers the changed header and shorter forms in both themes; broader live-provider and repository-wide acceptance remains separate. + +Final Settings verification: four additional interaction/accessibility checks passed for the corrected Conversations empty state. Reviewed 70 changed candidates and both new long-header theme snapshots; after correcting the stale empty-state instruction, all 90 final affected visual comparisons passed against the reviewed local baseline directory (`/tmp/speko-settings-cleanup-visual-final.log`). No unrelated snapshots were updated. The final UI/Storybook builds passed; the dev server was reloaded and the saved connection Settings verified in the embedded browser. Linux baseline publication and whole-feature qualification remain separate. + +## October 8: incoming conversation on its own low-trust task + +New calls admitted with **Let callers start new low-trust tasks without sign-in** now create a task before greeting, bind every spoken request/follow-up to that task, and retrieve its approved replies through the normal chat queue. The legacy `guestIntake` wire name remains; it no longer means message-only intake for new calls. Defaults stay private; existing authenticated task approval remains available. Caller ID is never a user identity. + +For E09, configure an active sandbox on the incoming line (or use the selected agent's sandbox), enable isolated workspaces, and supply an approved agent-scoped model connection. This affects only new low-trust incoming tasks; it does not change the agent/instance default environment or historical task assignments. Missing isolation fails closed. Browser/outgoing task conversations keep their existing authenticated authority. + +1. Call the configured company number from a telephone. Verify the greeting asks what to work on without a Paperclip code or sign-in. Confirm exactly one new task/session/admission. +2. Submit work, interrupt playback, and add a follow-up. Verify both requests and approved agent replies remain on that task, in order. A playback interruption does not cancel work. +3. Ask for another private task or a governed approval. Verify the low-trust boundary refuses it. Do not expand privileges or copy a caller-ID identity to make the test pass. +4. End the call. The task and already accepted work survive hangup; new call requests fail. When Speko's report arrives, verify the task's **Phone conversation transcript** document contains both caller and voice-agent turns, interruptions included, with quarantined source trust. Repeated reports must not create duplicate documents or revisions. No recording is downloaded. +5. Repeat with connection pause/removal, credential rotation, altered task boundary/assignment or sandbox selection. Current authority must be checked before queue processing and reply retrieval. + +Daytona cold sandboxes bootstrap the adapter's default Claude ACP server without changing engines; custom ACP commands remain setup failures if unavailable. A changed callback origin can update only tools/webhooks whose exact old definition and provider ID have completed receipts for this endpoint. Foreign integrations and uncertain creation outcomes remain blocked. For a quick Cloudflare test tunnel, expose only signed `POST /api/voice-webhooks/{publicId}/tools` and `/events`; the board API must return 404. Reconnect through the normal connection API after changing the explicit HTTPS origin. + +The [dated incoming qualification](2026-10-09-speko-pr-qualification.md) distinguishes signed callback tests, actual Daytona/Claude execution and real telephone audio. Its passing backend proof does not qualify human incoming audio, Safari or the 60-second telephone acceptance case. The old insufficient-credit blocker is historical: check current Speko usage before making acceptance calls. + + +## October 8: live-call context and dual-model handoff + +The voice model owns fast acknowledgments, clarification, and turn taking. The task agent owns work and substantive results. New public phone tasks capture the normal Speko communication guidance plus their low-trust boundary. Accepted, active voice sessions add a server-derived live-call instruction to every execution turn, including resumed sessions. Ending or expiring the session removes that instruction on the next turn; accepted work continues normally. + +`get_updates` retains its durable publication cursor and adds a neutral `work` projection: idle, queued, running, pending, awaiting_reply, waiting_for_input, blocked, failed, or cancelled. These are queue/execution observations, not task completion claims or spoken playback receipts. Raw run output and provider errors are excluded. Tool retries retain the original response; use a new tool-call ID for a new status observation. + +Run the disk scenario against the existing 10 GiB Daytona test task with locally generated `disk-request.pcm` and `disk-followup.pcm` (48 kHz mono signed 16-bit PCM): + +```sh +SPEKO_NATIVE_PORT=3469 \ +SPEKO_NATIVE_SCENARIO=disk \ +SPEKO_NATIVE_RESUME_ISSUE_ID= \ +SPEKO_NATIVE_FIXTURE_PATH= \ +SPEKO_NATIVE_AUDIO_DIR= \ +node tests/speko-native/live-browser.mjs +``` + +The fixture specifies companyId, companyPrefix, agentId, agentName, endpointId, and providerAgentId. Use an authorized disposable task already pinned to the real Daytona environment. The current disk scenario expects the test environment's 10 GiB root disk and approximately 9.7 GiB free; it is not a generic capacity test for arbitrary agents. For delayed delivery, the first recording requests a 65-second wait before inspecting only the root filesystem; the follow-up requests a one-sentence answer. Verify actual execution duration, two accepted submit_request receipts, same-task comments, urgent context in all task-markdown variants, approved publications, media cleanup, and synthetic output audio independently. A browser cascade result does not qualify telephone speech-to-speech or PSTN transport. diff --git a/doc/plans/2026-10-09-speko-pr-qualification.md b/doc/plans/2026-10-09-speko-pr-qualification.md new file mode 100644 index 00000000000..4066d2faeaa --- /dev/null +++ b/doc/plans/2026-10-09-speko-pr-qualification.md @@ -0,0 +1,99 @@ +# Speko review qualification — 2026-10-09 + +Status: experimental. Deterministic checks and real-provider acceptance are separate gates. + +## Review stack + +- Runtime and task/session authority: PR #15727. +- Production UI and deterministic Storybook fixtures: PR #15728. +- This qualification PR adds integrated tests, protocol proof tools, visual/a11y coverage, and the E01–E16 runbook. + +## Current evidence + +107 targeted migration, provider, protocol, and session-store tests passed after integrating current master. Fresh installation, pre-Speko upgrade, and repeated migration application passed. Full repository typecheck and build passed on the master-integrated feature before the review repairs. The full local test run hit timeouts in workspace snapshot, runtime, runner API, and native-session suites on a heavily loaded host (load average exceeded 80); it was stopped after those failures. This is not a passing full local test result. Latest-head CI must pass independently. The final stack includes runtime 82a96b128d; UI 43be85d01 was fully green before inheriting this test-only fixture change. The final original-feature checkpoint is 720379c17. + +Fresh review-repair evidence: + +| Gate | Result | Tested source / evidence | +| --- | --- | --- | +| Durable session, authority, reply, question and admission regressions | Passed: 75 tests | Runtime 99fa4a2e85; `/tmp/speko-pr-admission-tests-current.log` | +| UI call recovery, controller and readable history after errors | Passed: 35 tests | UI dadb0f106 plus test-only TypeScript signature correction; `/tmp/speko-pr-ui-history-tests.log` | +| UI typecheck after repairs | Passed | UI 2caea58bad; `/tmp/speko-pr-ui-history-typecheck-final.log` | +| Post-master migration upgrade/replay and heartbeat scheduling/preparation | Passed: 27 tests | Runtime bf1ead3772; `/tmp/speko-pr-post-master-tests.log` | +| Server typecheck after master integration | Passed | Runtime bf1ead3772; `/tmp/speko-pr-post-master-typecheck.log` | +| Heartbeat preparation and communication guidance | Passed: 15 tests | Runtime 174d08bc64; `/tmp/speko-pr-guidance-tests.log` | +| Proof protocol and initial report redaction | Passed: 9 tests | `/tmp/speko-pr-proof-tests.log` | +| Cleanup-failure report redaction | Passed: 2 tests | `/tmp/speko-pr-proof-redaction-final.log` | +| Token gates | Passed | Combined feature review repairs; `/tmp/speko-pr-token-final.log` | +| Storybook build / interaction / accessibility / responsive capture | Passed: 148 stories, 592 checks | Light/dark, 390/1200 width, reduced motion; `/tmp/speko-pr-storybook-checks.log` | +| Added history-error stories and affected pages | Passed: 188 checks | Fresh 150-story build; `/tmp/speko-pr-storybook-affected.log`; two secondary-error stories included | +| Task voice request error/retry and active-control preservation | Passed: 3 regressions, UI typecheck and token gates | UI 845e622b06; `/tmp/speko-pr-task-launcher-tests.log`, `/tmp/speko-pr-task-launcher-typecheck.log`, `/tmp/speko-pr-task-token.log` | +| Bounded notification fetch and request coalescing recovery | Passed: 13 API tests | UI 38057142bb; `/tmp/speko-pr-notification-timeout-tests.log` | +| Multipart answer delivery, ordering and Backlog admission | Passed: 78 persistence tests and server typecheck | Runtime 04a69ea6a; `/tmp/speko-pr-ordered-persistence-final.log`, `/tmp/speko-pr-runtime-exact-final-typecheck.log` | +| Final UI request recovery and footer | Passed: 6 regressions, typecheck, token gates; 2 focus assertions rerun | `/tmp/speko-pr-ui-review-final-tests.log`, `/tmp/speko-pr-ui-review-final-typecheck.log`, `/tmp/speko-pr-ui-review-token-final.log`, `/tmp/speko-pr-incoming-focus-tests.log` | +| Final setup, task launcher and incoming-call recovery stories | Passed: 153-story build, 220 affected checks | UI 43be85d01 with runtime 04a69ea6a; `/tmp/speko-pr-storybook-all-review-final.log`, `/tmp/speko-pr-storybook-all-review-checks.log`; no unrelated baseline updates | +| Authenticated E03/E14 isolation and revocation | Passed: 1 journey | Feature f121270c8; `/tmp/speko-pr-authenticated-final.log` | +| Integrated browser/phone journeys | Two passed; main journey failed in latest retry | `/tmp/speko-pr-integrated-startup-budget-final.log`; initial browser transcript did not retrieve saved answers. Execution and publication success do not supersede the visible delivery failure. Earlier successful portions are historical. | +| Dependency locking | Repository automation | The user explicitly requested no lockfiles. Root locking is managed by the existing refresh-lockfile workflow; standalone proof installation uses --no-package-lock. | + +The master-integrated combined source is preserved in feature commit 134d5b67c (origin/master 98e3cec78e); the successful authenticated run used the prior feature commit f121270c8. At that revision Speko migrations were 0323–0329 after master added 0322. Earlier integrated attempts ran the preceding feature commit with the review patches applied. Reports do not promote those runs to exact-head acceptance. + +The integrated test exposed a Slack-only communication-guidance join introduced by the master refactor. The runtime repair now delivers connection guidance to Speko and retains Slack-only command hints. The post-master fixture completed the benchmark in 66.9 seconds and saved approved replies; its browser trace showed the last notification GET stayed in flight until the assertion expired. UI 38057142bb adds a ten-second fetch deadline, with an abort-and-next-poll recovery test. A subsequent fresh run retrieved delayed browser answers, resumed work, answered structured questions, recovered across restart and invoked the assigned outbound-call tool; its inbound execution assertion failed because the fixture selected a Backlog task, which the execution gate correctly refused. The fixture now selects a Todo task, and submit_request immediately rejects Backlog work with an actionable message. The next fresh run retrieved browser replies but failed its 60-second restart-evidence assertion; the new app later served both shorter journeys, which passed (`/tmp/speko-pr-integrated-reviewed-final.log`). Restart now uses the same 120-second startup budget as initial fixture boot, retaining the process-ID and recovery assertions. The final fresh retry failed earlier at the initial browser-answer assertion (100 seconds); both shorter journeys passed (`/tmp/speko-pr-integrated-startup-budget-final.log`). Its synthetic runtime persisted two approved replies, but neither was retrieved in the browser. The request trace shows notification requests returning null, followed by a request at 21:45:57 UTC that remained unfinished through failure; the loaded production bundle contains the ten-second fetch deadline. This remains an unresolved integration failure; host contention alone is not a demonstrated cause. No inbound-push or restart qualification is claimed for this attempt. A local 600-second overall budget is used on this contended host; the repository 360-second default remains unchanged. Healthy-provider timing targets remain independently unqualified. A further retry reused the previous provider fixture and failed setup, so it is not acceptance evidence. Temporary test services were shut down without changing the saved user instance. + +The new long-answer repair splits provider messages at 16,000 UTF-16 code units without cutting surrogate pairs, persists each part receipt, resumes known rate-limit failures, and blocks polling from replaying an accepted prefix. The 77-test persistence suite initially hit its explicit 30-second database startup hook before any assertions ran on the loaded host. It now uses the repository’s documented embedded-Postgres test startup budget. All 77 tests passed on runtime a7002737b, including multipart rate-limit/restart and unknown-outcome regressions and protection against polling replay of an accepted prefix (`/tmp/speko-pr-multipart-persistence-final.log`). Server typecheck passed (`/tmp/speko-pr-multipart-final-typecheck.log`). + +Runtime 04a69ea6a received Greptile 5/5 with no unresolved review threads, and its CI typecheck/build passed. A chat-retry browser shard failed its authorization-denial feedback assertion twice on runtime 04a69ea6a. Runtime 82a96b128d begins that fixture directly on the canonical agent route, preserving authority and visible-error assertions. Its three-repeat local verification was blocked before tests ran by the 120-second server startup timeout (`/tmp/speko-pr-chat-retry-canonical.log`); all ordinary CI checks subsequently passed. UI 43be85d01 passed its ordinary CI tests/build/Canary and received Greptile 5/5; dependency locking follows the existing repository automation. The user explicitly requested that no lockfiles be added. + +A real OpenCode/OpenRouter DeepSeek task executed in the configured Daytona sandbox, preserved low_trust_review, ran df -h, and persisted its initial update and final answer. Its 10 GB root overlay had about 9.9 GB available. The ephemeral sandbox cleanup succeeded. + +The initial task update took 33.3 seconds, the final answer 44.3 seconds, and run completion 80.5 seconds from task start. These are cold sandbox timings, not speech latency. They do not meet the two-second live acknowledgment target. + +## Outstanding failures and unverified behavior + +The remote run succeeded but task settlement retained in_progress: its completion claim used bare comment UUIDs as evidence and supplied no durable verification reference. The status arbiter rejected that evidence. This is a runtime completion-evidence issue, not an answer or sandbox startup success. + +The original call had ended before that board retry. There was no new voice reply or publication for the retry. Provider push acceptance and spoken playback were not proven. Older publication records belong to the earlier failed call and must not be used to qualify this retry. + +Browser voice, inbound telephone, outbound callback, Safari/mobile playback, and 60-second delayed-result delivery need the real-provider acceptance matrix. Test fixtures do not substitute for actual audio. No new phone call, speech input, recording download, or participant audio capture is part of PR preparation. + +## Evidence handling + +Private call logs, provider account IDs, credentials, and transcripts remain outside these public PRs. The test harness writes new evidence to local report directories. Only consented, redacted results should be shared. Historical development reports are retained in the original feature worktree; they are not current-head proof. + +See [the E2E runbook](2026-09-11-speko-voice-e2e-runbook.md) and [the component/story inventory](2026-09-11-speko-storybook-inventory.md). Keep Speko behind the existing experimental chat-connector setting until all supported real-provider journeys pass. + +### Final merge preparation + +The user authorized merging the feature and explicitly requested no lockfiles. The standalone proof lockfile was removed and its installation instructions updated. The final cleanup repairs preserve reports on failed provider hangup, allow retrying unconfirmed hangup, coalesce concurrent cleanup, and continue media cleanup after failures. Five deterministic cleanup/redaction tests passed. The connected live-browser harness now creates its qualification task in Todo, where submit_request can run. Syntax checks passed; no live calls were placed during merge preparation. Master integration renumbers Speko migrations to 0324–0330 and retains the new GitHub registration migration. Exact-head CI and review are checked before merge. + +The final master-integrated runtime c7b00b5988 passed the real embedded-Postgres upgrade/replay test, 42 communication-guidance/provider/protocol tests, database/shared builds, and server typecheck. UI 7c7c2b38c6 passed typecheck, token gates, 26 focused component/connect-policy tests, and the full 154-story Speko Storybook build. All 108 affected approval and conversation interaction/accessibility checks passed in light/dark themes at 390/1200 widths with reduced motion, real microphone access forbidden, and external networking blocked (`/tmp/speko-final-merge-storybook-checks.log`). These deterministic passes do not resolve the separately documented integrated browser-answer retrieval failure or establish live audio acceptance. + +The changed approved-call confirmation was visually inspected in both light and dark narrow layouts. No unrelated visual baselines were updated. + +### Runtime authority review repairs + +Runtime f03a42673 (carried in master-integrated 7e73d5ba21) fixes three new review findings. Both missed-reply collection and publication retry check prior-call receipts for the same caller, endpoint, task, conversation, and publication; a partly accepted or uncertain answer cannot automatically replay on rejoin. Callback preference writes no longer install agent tools: verified connection setup provisions them only after normal agent_config:update authorization. Guest intake clarifications use task comments and submit_request follow-ups; agent-created and system-created protected question waits are rejected for those unverified tasks. + +All 83 real-database voice persistence/authority regressions passed (`/tmp/speko-final-review-security-tests-retry.log`); a focused final test also passed for both agent/system question creators (`/tmp/speko-final-guest-native-guard.log`). Server typecheck passed. The first local database attempt skipped all tests after its startup probe failed; that skipped run was not counted as passing. The retry used a successful real embedded-Postgres instance. Fresh exact-head CI/review are required before merging. No provider calls were placed, and no lockfiles were added. + +Latest UI review repair: voice start, inspection, and hangup requests now have ten-second deadlines. Production-client/controller regressions prove that a lost start retains its idempotency key across reload, and an uncertain hangup retains the exact call for cleanup retry without reopening media. All 36 focused API/lifecycle/recovery tests, UI typecheck, and token gates passed. No UI component markup or lockfiles changed. + +Runtime follow-up review repairs preserve pending questions across reconnect, allow canonical authenticated task-page answers to continue the original voice request, and retain accepted guest tasks in their saved sandbox when future-call settings change. Current permission, credential, isolation, sandbox availability, and company ownership checks remain enforced. All 85 voice persistence tests and server typecheck passed. Five focused native/legacy signed-tool and task-page continuation/authority regressions passed; the other 145 cases were intentionally unselected in that focused run. + +Harness review repairs: successful local cleanup operations are tracked individually; failed operations remain retryable and End stays available until local and provider cleanup complete. Eight cleanup/redaction tests pass. Audio cleanup awaits finalized MediaRecorder data and a successful upload; a failed upload retains the recording for retry and keeps End available. Recording finalization and upload have bounded deadlines. No real audio session was run for this repair. The live browser harness attempts provider hangup independently of evidence-write failures. README links now point to the included report and runbook instead of absent historical private reports. + +Interrupted setup qualification: safe secret-presence serialization and production-component recovery regressions pass, alongside UI/server typecheck and token gates. The 155-story build and all 32 affected provider-setup interaction/accessibility checks passed across both themes and desktop/mobile widths. + +Callback rotation review repair: ownership receipts now bind the provider identity and exact old callback independently from the replacement secret. Legacy receipts use the previous vaulted key and are upgraded before provider changes. Five focused real-database ownership/rotation/uncertain-creation tests and server typecheck passed; current and legacy simultaneous URL/key rotation and foreign callback rejection are covered. + +Qualification provenance and media repair: the live browser report reads the assigned agent’s actual adapter type instead of inferring Claude from the scenario name. Synthetic media pump failures are observed immediately, release blocked input, and enter the hangup/evidence finally path. Ten focused cleanup/media/redaction tests pass; this repair placed no live calls. + +October 13 review repair: browser evidence saves are serialized, with each report snapshotted when enqueued. Final cleanup evidence waits for any prior save, and a failed periodic write does not prevent the final report. All 20 standalone proof tests passed, including the local HTTP surface; browser-client syntax check passed. No real calls were placed. The UI repairs passed 26 focused controller/launcher tests, UI typecheck, and token gates. Fresh CI and exact-head review are required before merging the repaired heads. + +October 13 master reconciliation: all 76 affected Storybook checks passed and narrow-screen captures were inspected. UI master integration retains the generated-catalog policy and adds source-owned Speko capability annotations; 28 map structural tests, 26 focused UI tests, UI typecheck and token gates pass. An unchanged skill-audit deadline test failed once in UI CI, passed on the identical qualification backend and locally, and its single failed-job retry passed. No assertions or deadlines were weakened. + +Final October 13 fixture evidence: all 624 Speko Storybook interaction/accessibility checks passed across 156 stories, both themes, 390/1200 widths and reduced motion on UI source 249cbc616a. The subsequent merge adds GitHub server guidance and documentation without changing Speko UI source. Narrow captures were inspected. Report requests now have ten-second abort deadlines, periodic saves cannot build a backlog, and server disk writes remain serialized after an aborted client request. All 21 proof tests passed, including stalled transport followed by a late server write and final evidence preservation. Browser client/server syntax checks passed. Live-call acceptance remains unproven. + +October 13 merge handoff: UI #15728 merged at 22d05644f4888ff75c344c471524acf993087338 after 55 passing checks and a fresh 5/5 review with no unresolved findings. Its final source b5684d9a42 passed all 152 affected page/launcher Storybook checks across both themes and narrow/desktop widths; the final build contains 157 Speko stories. Qualification is refreshed against merged master. The visual runner preserves both configurable local ports and the upstream external Storybook base URL override. No provider calls or lockfiles were added. + +Hosted Storybook review repair: the Speko request guard permits only the configured Storybook origin, including hosted URLs, and continues to reject unrelated provider requests. A deterministic browser regression serves a synthetic hosted origin without external traffic and verifies navigation/assets succeed while Speko requests are blocked. Hosted-origin isolation and representative local narrow-screen stories pass. Two CI jobs on the prior head were interrupted by runner shutdown signals, without assertion failures; the repair requires fresh current-head CI and review. diff --git a/tests/speko-native/.gitignore b/tests/speko-native/.gitignore new file mode 100644 index 00000000000..8a194ebdb8b --- /dev/null +++ b/tests/speko-native/.gitignore @@ -0,0 +1,8 @@ +.instance.json +test-results/ +playwright-report/ + +.auth-instance.json +test-results-auth/ +playwright-report-auth/ +qualification/ diff --git a/tests/speko-native/app.ts b/tests/speko-native/app.ts new file mode 100644 index 00000000000..8154f6cce89 --- /dev/null +++ b/tests/speko-native/app.ts @@ -0,0 +1,164 @@ +/** Production app with a synthetic execution provider; launched by server.ts. */ +import { appendFile, writeFile, rename } from "node:fs/promises"; +import { resolve } from "node:path"; +import { randomInt } from "node:crypto"; +import { once } from "node:events"; +import { createRequire } from "node:module"; +import { setTimeout as delay } from "node:timers/promises"; +import { createDb, authUsers, instanceUserRoles, issueComments, issues, issueThreadInteractions, issueQuestionResponseDeliveries, chatVoiceInboundCalls, chatVoiceReports, chatVoiceSessions, chatVoiceReplies, chatVoiceToolCalls, chatActions, chatDeliveries, chatPublications, heartbeatRuns, agentWakeupRequests } from "../../packages/db/src/index.js"; +const home = process.env.PAPERCLIP_HOME!; +const port = Number(process.env.SPEKO_NATIVE_PORT ?? 3449); +const origin = `http://127.0.0.1:${port}`; +const callbackOrigin = process.env.SPEKO_NATIVE_CALLBACK_ORIGIN!; +const requireServer = createRequire(new URL("../../server/package.json", import.meta.url)); +const { eq } = requireServer("drizzle-orm"); +const db = createDb(process.env.DATABASE_URL!); +const authenticated = process.env.SPEKO_NATIVE_AUTH === "1"; +const deploymentMode = authenticated ? "authenticated" : "local_trusted"; +// Match index.ts's local-trusted bootstrap: agent JWTs retain this real +// responsible user and the memberships created by the normal company route. +if (!authenticated) await db.insert(authUsers).values({ id: "local-board", name: "Board", email: "local@paperclip.local", emailVerified: true, createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); +if (!authenticated) await db.insert(instanceUserRoles).values({ userId: "local-board", role: "instance_admin" }).onConflictDoNothing(); +const { registerServerAdapter, waitForExternalAdapters } = await import("../../server/src/adapters/registry.js"); +await waitForExternalAdapters(); +const { instanceSettingsService } = await import("../../server/src/services/instance-settings.js"); +await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); +const firstRun = new Map(); +registerServerAdapter({ + type: "process", supportsLocalAgentJwt: true, + async testEnvironment() { return { adapterType: "process", status: "pass", checks: [], testedAt: new Date().toISOString() }; }, + async execute(ctx) { + const issueId = String(ctx.context.issueId ?? ""); + if (!issueId || !ctx.authToken) throw new Error("Expected real task and run-scoped agent identity"); + ctx.onDispatch?.(); + await ctx.onCancellationReady?.(); + if (ctx.config.command === "fixture-phone") { + // Only execution is synthetic. Discover and invoke through the real + // run-scoped MCP transport; do not reach into the voice service. + const server = ctx.runtimeMcp?.getServers().find(server => server.name === "paperclip-assigned"); + if (!server) throw new Error("The assigned phone tool was not delivered to the runtime"); + const { Client } = requireServer("@modelcontextprotocol/sdk/client/index.js"); + const { StreamableHTTPClientTransport } = requireServer("@modelcontextprotocol/sdk/client/streamableHttp.js"); + const client = new Client({ name: "speko-phone-fixture", version: "1.0.0" }); + try { + await client.connect(new StreamableHTTPClientTransport(new URL(server.url), { requestInit: { headers: { authorization: `Bearer ${server.token}` } } })); + const listed = await client.listTools(); + const tool = listed.tools.find((tool: { name: string }) => tool.name.endsWith(":call_my_phone")); + if (!tool) throw new Error("The callback tool was not discoverable"); + const result = await client.callTool({ name: tool.name, arguments: {} }); + if (result.isError) throw new Error("The fixture phone call failed"); + // A second click must not dial again while this exact task is on a call. + let secondRejected = false; + try { const repeated = await client.callTool({ name: tool.name, arguments: {} }); secondRejected = !!repeated.isError; } catch { secondRejected = true; } + if (!secondRejected) throw new Error("A simultaneous duplicate phone request was not rejected"); + await appendFile(resolve(home, "execution.jsonl"), JSON.stringify({ event: "phone_called", issueId, runId: ctx.runId, result, secondRejected }) + "\n", { mode: 0o600 }); + return { exitCode: 0, signal: null, timedOut: false, summary: "The assigned phone tool placed one call; the duplicate request was rejected." }; + } finally { await client.close(); } + } + let run = firstRun.get(issueId); + const followupRun = !!run; + if (!run) { run = { began: Date.now(), number: randomInt(1000, 10000) }; firstRun.set(issueId, run); } + await appendFile(resolve(home, "execution.jsonl"), JSON.stringify({ event: "started", issueId, runId: ctx.runId, at: new Date().toISOString() }) + "\n", { mode: 0o600 }); + await ctx.onLog("stdout", "Synthetic execution provider is processing the voice request.\n"); + const remaining = run.began + 61_000 - Date.now(); + if (remaining > 0) await delay(remaining, undefined, { signal: ctx.signal }); + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + const requests = comments.filter((comment) => !comment.authorAgentId); + if (requests.some((comment) => comment.body.includes("Ask the structured acceptance question"))) { + const interactions = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId)); + const question = interactions.find((item) => item.idempotencyKey === `fixture-question:${issueId}`); + if (!question) { + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + const response = await fetch(`${origin}/api/issues/${issueId}/interactions`, { + method: "POST", headers: { authorization: `Bearer ${ctx.authToken}`, "content-type": "application/json", "x-paperclip-run-id": ctx.runId }, + body: JSON.stringify({ kind: "ask_user_questions", idempotencyKey: `fixture-question:${issueId}`, addresseeUserId: task.responsibleUserId, + payload: { version: 1, questions: [{ id: "color", prompt: "Which color should I use?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "cobalt", label: "Cobalt" }, { id: "amber", label: "Amber" }] }] } }), signal: ctx.signal, + }); + if (!response.ok) throw new Error(`Fixture question API rejected: ${response.status}`); + return { exitCode: 0, signal: null, timedOut: false, summary: "" }; + } + if (question.status !== "answered") return { exitCode: 0, signal: null, timedOut: false, summary: "" }; + } + const answeredQuestion = (await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).find((item) => item.idempotencyKey === `fixture-question:${issueId}` && item.status === "answered"); + const summary = answeredQuestion ? `Your structured answer was received: ${JSON.stringify(answeredQuestion.result)}.` : followupRun ? "Your follow-up was included in the completed result." : `The delayed work is complete. Verification number ${run.number}. I received ${Math.max(0, requests.length - 1)} follow-up instructions on this same task.`; + const response = await fetch(`${origin}/api/issues/${issueId}/comments`, { + method: "POST", headers: { authorization: `Bearer ${ctx.authToken}`, "content-type": "application/json", "x-paperclip-run-id": ctx.runId }, + body: JSON.stringify({ body: summary }), signal: ctx.signal, + }); + if (!response.ok) throw new Error(`Normal agent comment API rejected fixture: ${response.status} ${(await response.json() as { error: string }).error}`); + const comment = await response.json() as { id: string }; + await appendFile(resolve(home, "execution.jsonl"), JSON.stringify({ event: "completed", issueId, runId: ctx.runId, commentId: comment.id, number: run.number, requests: requests.length, elapsedMs: Date.now() - run.began, at: new Date().toISOString() }) + "\n", { mode: 0o600 }); + return { exitCode: 0, signal: null, timedOut: false, summary, resultJson: { summary } }; + }, +}); +const closeStub = process.env.SPEKO_NATIVE_STUB === "1" + ? await (await import("./stub-provider.js")).installSpekoTestProvider(origin, Number(process.env.SPEKO_NATIVE_STUB_PORT ?? port + 1)) + : undefined; +const { createApp } = await import("../../server/src/app.js"); +const { createStorageService } = await import("../../server/src/storage/service.js"); +const { createLocalDiskStorageProvider } = await import("../../server/src/storage/local-disk-provider.js"); +const authModule = await import("../../server/src/auth/better-auth.js"); +const { loadConfig } = await import("../../server/src/config.js"); +const auth = authenticated ? authModule.createBetterAuthInstance(db, loadConfig(), [origin]) : undefined; +const app = await createApp(db, { + uiMode: "static", serverPort: port, bindHost: "127.0.0.1", allowedHostnames: ["127.0.0.1", "localhost", new URL(callbackOrigin).hostname], + deploymentMode, + ...(auth ? { betterAuthHandler: authModule.createBetterAuthHandler(auth), resolveSession: (req: import("express").Request) => authModule.resolveBetterAuthSession(auth, req) } : {}), + deploymentExposure: "private", authReady: true, companyDeletionEnabled: true, + chatWebhookPublicBaseUrl: callbackOrigin, authPublicBaseUrl: origin, + storageService: createStorageService(createLocalDiskStorageProvider(resolve(home, "storage"))), + decisionServiceOptions: { wakeOriginAgent: async () => {} }, managedPluginAutoInstall: [], +}); +let evidenceWriting = false; +async function writeEvidence() { + if (evidenceWriting) return; + evidenceWriting = true; + try { + const [reports, incoming, sessions, replies, tools, deliveries, publications, runs, wakeups, comments, interactions, questionDeliveries, pushActions] = await Promise.all([ + db.select({sessionId: chatVoiceReports.sessionId, companyId: chatVoiceReports.companyId, status: chatVoiceReports.status, costMicroUsd: chatVoiceReports.costMicroUsd, transcript: chatVoiceReports.transcript}).from(chatVoiceReports), + db.select({id: chatVoiceInboundCalls.id, companyId: chatVoiceInboundCalls.companyId, endpointId: chatVoiceInboundCalls.endpointId, state: chatVoiceInboundCalls.state, sessionId: chatVoiceInboundCalls.sessionId}).from(chatVoiceInboundCalls), + db.select({ id: chatVoiceSessions.id, issueId: chatVoiceSessions.issueId, providerSessionId: chatVoiceSessions.providerSessionId, mode: chatVoiceSessions.mode, state: chatVoiceSessions.state }).from(chatVoiceSessions), + db.select().from(chatVoiceReplies), + db.select({ id: chatVoiceToolCalls.id, sessionId: chatVoiceToolCalls.sessionId, tool: chatVoiceToolCalls.tool, response: chatVoiceToolCalls.response }).from(chatVoiceToolCalls), + db.select({ id: chatDeliveries.id, state: chatDeliveries.state, conversationId: chatDeliveries.conversationId, redactedError: chatDeliveries.redactedError }).from(chatDeliveries), + db.select({ id: chatPublications.id, issueId: chatPublications.issueId, state: chatPublications.state, payload: chatPublications.payload, redactedError: chatPublications.redactedError }).from(chatPublications), + db.select({ id: heartbeatRuns.id, agentId: heartbeatRuns.agentId, status: heartbeatRuns.status, error: heartbeatRuns.error, errorCode: heartbeatRuns.errorCode, runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId, contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns), + db.select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status, runId: agentWakeupRequests.runId }).from(agentWakeupRequests), + db.select({ id: issueComments.id, issueId: issueComments.issueId, body: issueComments.body, authorAgentId: issueComments.authorAgentId, createdByRunId: issueComments.createdByRunId }).from(issueComments), + db.select().from(issueThreadInteractions), + db.select().from(issueQuestionResponseDeliveries), + db.select().from(chatActions).where(eq(chatActions.kind, "speko_voice_reply_push")), + ]); + const runEvidence = runs.map(({ contextSnapshot, ...row }) => { + const context = (contextSnapshot ?? {}) as Record; + const wake = (context.paperclipWake ?? {}) as Record; + return { ...row, context: { source: context.source, issueId: context.issueId, wakeCommentIds: context.wakeCommentIds, + paperclipTaskCommunicationGuidance: context.paperclipTaskCommunicationGuidance, paperclipHarnessCheckedOut: context.paperclipHarnessCheckedOut, paperclipExternalChatExecutionBound: context.paperclipExternalChatExecutionBound, + wake: { externalChatProvider: wake.externalChatProvider, checkedOutByHarness: wake.checkedOutByHarness, externalChatExecutionBound: wake.externalChatExecutionBound } } }; + }); + await writeFile(resolve(home, "evidence.pending.json"), JSON.stringify({ appPid: process.pid, at: new Date().toISOString(), reports, incoming, sessions, replies, tools, deliveries, publications, runs: runEvidence, wakeups, comments, interactions, questionDeliveries, pushActions }), { mode: 0o600 }); + await rename(resolve(home, "evidence.pending.json"), resolve(home, "evidence.json")); + } finally { evidenceWriting = false; } +} +await writeEvidence(); +const evidenceTimer = setInterval(() => { void writeEvidence().catch((error) => console.error("Evidence snapshot failed:", error.message)); }, 2000); +evidenceTimer.unref(); +const { createPaperclipHttpServer } = await import("../../server/src/http/server.js"); +const server = createPaperclipHttpServer(app, { apiUrl: origin }); +server.listen(port, "127.0.0.1"); +const { setupLiveEventsWebSocketServer } = await import("../../server/src/realtime/live-events-ws.js"); +const liveEvents = setupLiveEventsWebSocketServer(server, db, { deploymentMode, ...(auth ? { resolveSessionFromHeaders: (headers: Headers) => authModule.resolveBetterAuthSessionFromHeaders(auth, headers) } : {}) }); +await once(server, "listening"); +console.log(JSON.stringify({ ready: true, origin, home })); +let stopping = false; +async function stop() { + if (stopping) return; stopping = true; + clearInterval(evidenceTimer); + for (const client of liveEvents.clients) client.terminate(); + server.close(); server.closeAllConnections(); + await app.locals.paperclipShutdown?.(); + await closeStub?.(); + process.exit(0); +} +process.once("SIGINT", () => void stop()); +process.once("SIGTERM", () => void stop()); diff --git a/tests/speko-native/authenticated.spec.ts b/tests/speko-native/authenticated.spec.ts new file mode 100644 index 00000000000..2ab517e125d --- /dev/null +++ b/tests/speko-native/authenticated.spec.ts @@ -0,0 +1,66 @@ +import { test, expect, type APIRequestContext } from "@playwright/test"; +import { randomBytes, randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +const origin = "http://127.0.0.1:3482"; + +test("E03/E14: real signed-in callers cannot substitute companies, private tasks or another caller; revoked members can hang up", async ({ browser, page }, info) => { + const memberContext = await browser.newContext(), outsiderContext = await browser.newContext(), anonymousContext = await browser.newContext(); + const request = async (api: APIRequestContext, method: "POST" | "PATCH", path: string, data: unknown) => { + const response = await api.fetch(`${origin}/api${path}`, { method, headers: { Origin: origin }, data }); + expect(response.ok(), `${method} ${path}: ${response.status()} ${await response.text()}`).toBe(true); + return response.json(); + }; + const signUp = (api: APIRequestContext, name: string) => request(api, "POST", "/auth/sign-up/email", { name, email: `${randomUUID()}@example.test`, password: randomBytes(24).toString("base64url") }); + try { + await signUp(page.request, "Voice operator"); + await request(page.request, "POST", "/bootstrap/claim", {}); + const member = await signUp(memberContext.request, "Voice member"); + await signUp(outsiderContext.request, "Other company member"); + const company = await request(page.request, "POST", "/companies", { name: "Private voice company" }); + const other = await request(page.request, "POST", "/companies", { name: "Other voice company" }); + const invite = async (api: APIRequestContext, companyId: string) => { + const invitation = await request(page.request, "POST", `/companies/${companyId}/invites`, { allowedJoinTypes: "human", humanRole: "operator" }); + const token = new URL(invitation.inviteUrl, origin).pathname.split("/").at(-1); + await request(api, "POST", `/invites/${token}/accept`, { requestType: "human" }); + }; + await invite(memberContext.request, company.id); await invite(outsiderContext.request, other.id); + await request(page.request, "PATCH", `/companies/${company.id}`, { requireBoardApprovalForNewAgents: false }); + const agent = await request(page.request, "POST", `/companies/${company.id}/agents`, { name: "Voice worker", role: "general", adapterType: "process", adapterConfig: { command: "fixture" }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }); + await expect.poll(async () => (await (await page.request.get(`${origin}/api/agents/${agent.id}`)).json()).status).toBe("idle"); + const endpoint = await request(memberContext.request, "POST", `/companies/${company.id}/chat-endpoints`, { provider: "speko", assignedAgentId: agent.id }); + const configured = await request(memberContext.request, "POST", `/chat-endpoints/${endpoint.id}/setup`, { action: "configure", credentials: { apiKey: "fixture-speko-key", agentId: "agent_member", signingSecret: `whsec_${randomBytes(32).toString("base64")}` } }); + expect(configured.setup.step).toBe("test"); + const privateTask = await request(page.request, "POST", `/companies/${company.id}/issues`, { title: "Private task", status: "backlog", assigneeAgentId: agent.id, visibility: "private" }); + const start = (api: APIRequestContext, companyId = company.id, issueId?: string) => api.post(`${origin}/api/companies/${companyId}/voice-sessions`, { headers: { Origin: origin }, data: { endpointId: endpoint.id, ...(issueId ? { issueId } : {}), idempotencyKey: randomUUID() } }); + expect([401, 403]).toContain((await start(anonymousContext.request)).status()); + expect((await start(outsiderContext.request)).status()).toBe(403); + expect([403, 404]).toContain((await start(memberContext.request, other.id)).status()); + expect([403, 404]).toContain((await start(memberContext.request, company.id, privateTask.id)).status()); + const created = await start(memberContext.request); + expect(created.status()).toBe(201); + const { session, media } = await created.json(); + expect(session.callerId).toBeUndefined(); // Private authority metadata is not projected into the browser session. + expect(media.transportToken).toMatch(/^fixture:/); + const path = `/api/companies/${company.id}/voice-sessions/${session.id}`; + expect((await page.request.get(origin + path)).status()).toBe(403); + expect((await outsiderContext.request.get(origin + path)).status()).toBe(403); + expect((await memberContext.request.get(origin + path)).status()).toBe(200); + const { members } = await (await page.request.get(`${origin}/api/companies/${company.id}/members`)).json(); + const membership = members.find((entry: any) => entry.principalId === member.user.id); + expect(membership).toBeTruthy(); + await request(page.request, "PATCH", `/companies/${company.id}/members/${membership.id}`, { status: "suspended" }); + expect((await memberContext.request.get(origin + path + "/notification")).status()).toBe(403); + expect((await memberContext.request.get(origin + path)).status()).toBe(403); + const ended = await request(memberContext.request, "POST", path.replace(/^\/api/, "") + "/end", {}); + expect(ended.state).toBe("ended"); + const { home } = JSON.parse(await readFile(resolve(import.meta.dirname, ".auth-instance.json"), "utf8")); + await expect.poll(async () => { + const evidence = JSON.parse(await readFile(resolve(home, "evidence.json"), "utf8")); + return { sessions: evidence.sessions.length, ended: evidence.sessions[0]?.state, accepted: evidence.tools.length }; + }).toEqual({ sessions: 1, ended: "ended", accepted: 0 }); + await info.attach("authority-results", { body: JSON.stringify({ companyId: company.id, otherCompanyId: other.id, sessionId: session.id, privateTaskId: privateTask.id, denied: ["anonymous", "cross-company", "private-task", "other-caller", "suspended-member"], revokedCallerHangup: "passed" }), contentType: "application/json" }); + } finally { + await Promise.all([memberContext.close(), outsiderContext.close(), anonymousContext.close()]); + } +}); diff --git a/tests/speko-native/fixtures/acceptance-benchmark.py b/tests/speko-native/fixtures/acceptance-benchmark.py new file mode 100644 index 00000000000..04256d89adb --- /dev/null +++ b/tests/speko-native/fixtures/acceptance-benchmark.py @@ -0,0 +1,23 @@ +"""Explicit live-acceptance workload: compute for 65 seconds, then read the fixture. + +Copy into the disposable agent workspace. Uses one CPU core and no network. +This runs inside the real selected agent; it does not replace the execution adapter. +""" +import hashlib +import json +from pathlib import Path +import time + +started = time.monotonic() +iterations = 0 +value = b"paperclip-speko-acceptance" +while time.monotonic() - started < 65: + for _ in range(1024): + value = hashlib.sha256(value).digest() + iterations += 1024 +print(json.dumps({ + "elapsedSeconds": time.monotonic() - started, + "iterations": iterations, + "digest": value.hex(), + "fixture": Path("ACCEPTANCE.txt").read_text().strip(), +}), flush=True) diff --git a/tests/speko-native/integrated.spec.ts b/tests/speko-native/integrated.spec.ts new file mode 100644 index 00000000000..3939e10793f --- /dev/null +++ b/tests/speko-native/integrated.spec.ts @@ -0,0 +1,364 @@ +import { test, expect } from "@playwright/test"; +import { readFile, readdir } from "node:fs/promises"; +import { resolve } from "node:path"; +const root = resolve(import.meta.dirname, "../.."); +const port = Number(process.env.SPEKO_NATIVE_TEST_PORT ?? 3480); +const origin = `http://127.0.0.1:${port}`; + +test("E01/E03–E11/E13/E15/E16: setup, delayed work, duplicate input, navigation, repeat, reload and hangup use real task ledgers", async ({ page }, info) => { + const { home, supervisorPid } = JSON.parse(await readFile(resolve(import.meta.dirname, ".instance.json"), "utf8")); + const evidence = async () => JSON.parse(await readFile(resolve(home, "evidence.json"), "utf8")); + const post = async (path: string, data: unknown) => { + const response = await page.request.post(`${origin}/api${path}`, { headers: { Origin: origin }, data }); + expect(response.ok(), `${path}: ${response.status()} ${await response.text()}`).toBe(true); + return response.json(); + }; + const company = await post("/companies", { name: "Speko integrated", issuePrefix: "SPE" }); + const other = await post("/companies", { name: "Separate company", issuePrefix: "OTH" }); + const policy = await page.request.patch(`${origin}/api/companies/${company.id}`, { headers: { Origin: origin }, data: { requireBoardApprovalForNewAgents: false } }); + expect(policy.ok()).toBe(true); + const agent = await post(`/companies/${company.id}/agents`, { name: "Existing Paperclip Agent", role: "general", adapterType: "process", adapterConfig: { command: "fixture" }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }); + // Discover the built SDK chunk by its export, rather than mock any Paperclip API. + const assets = resolve(root, "ui/dist/assets"); + const candidates = await Promise.all((await readdir(assets)).filter(name => name.endsWith(".js")).map(async name => ({ name, source: await readFile(resolve(assets, name), "utf8") }))); + const sdk = candidates.filter(file => /export\{[^}]* as VoiceConversation\}/.test(file.source)); + expect(sdk).toHaveLength(1); + const sdkSource = (await readFile(resolve(import.meta.dirname, "stub-browser-sdk.mjs"), "utf8")).replaceAll("__STUB_PORT__", String(port + 1)); + await page.route(`**/assets/${sdk[0].name}`, route => route.fulfill({ contentType: "application/javascript", body: sdkSource })); + const external: string[] = []; + await page.route(/^https?:\/\//, route => { + const url = new URL(route.request().url()); + return ["127.0.0.1", "localhost"].includes(url.hostname) ? route.fallback() : route.abort(); + }); + page.on("request", request => { const url = new URL(request.url()); if (["http:", "https:", "wss:"].includes(url.protocol) && !["127.0.0.1", "localhost"].includes(url.hostname)) external.push(url.hostname); }); + await page.addInitScript(() => { navigator.mediaDevices.getUserMedia = async () => { throw new Error("Integrated fixture must not request a real microphone"); }; }); + await page.goto(`/${company.issuePrefix}/apps/chat/connect?provider=speko`); + await page.getByRole("button", { name: "Choose an active agent", exact: true }).click(); + await page.getByRole("button", { name: `Select ${agent.name}`, exact: true }).click(); + await page.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(page.getByLabel("Speko API key", { exact: true })).toHaveAttribute("type", "password"); + await page.getByRole("textbox", { name: "Speko agent ID", exact: true }).fill("agent_fixture"); + await page.getByLabel("Speko API key", { exact: true }).fill("fixture-speko-key"); + const setupResponse = page.waitForResponse(r => r.request().method() === "POST" && /\/chat-endpoints\/[^/]+\/setup$/.test(new URL(r.url()).pathname)); + await page.getByRole("button", { name: "Connect Speko", exact: true }).click(); + const endpoint = await (await setupResponse).json(); expect(endpoint.id).toBeTruthy(); + expect(JSON.stringify(endpoint)).not.toContain("fixture-speko-key"); + await page.reload(); + await expect(page.getByRole("button", { name: "Start voice", exact: true })).toBeVisible(); + const nextSession = () => page.waitForResponse(r => r.request().method() === "POST" && /\/voice-sessions$/.test(new URL(r.url()).pathname) && r.ok()).then(r => r.json()); + const pending = nextSession(); await page.getByRole("button", { name: "Start voice", exact: true }).click(); const started = await pending; + expect(started.session.assignedAgentId).toBe(agent.id); expect(started.media.transportToken).toMatch(/^fixture:/); + expect(JSON.stringify(started)).not.toContain("fixture-speko-key"); + await expect(page.getByRole("button", { name: "Mute", exact: true })).toBeEnabled(); + const speak = (text: string, id: string) => page.evaluate(({ text, id }) => (window as any).__spekoTransport.speak(text, id), { text, id }); + const accepted = await speak("Run the acceptance benchmark and report its verification number.", "request-one"); + expect(accepted.status).toBe("accepted"); + expect(await speak("Run the acceptance benchmark and report its verification number.", "request-one")).toEqual(accepted); + await speak("Also count this follow-up on the same task.", "request-two"); + // A client-side route transition must preserve the host and the original media. + await page.getByRole("link", { name: "View conversation task", exact: true }).click(); + await expect(page).toHaveURL(new RegExp(`/issues/${started.session.issueId}$`)); + await expect(page.getByRole("button", { name: "Mute", exact: true })).toBeEnabled(); + expect(await page.evaluate(() => (window as any).__spekoTransport.creates)).toBe(1); + await expect.poll(async () => (await evidence()).tools.filter((tool: any) => tool.sessionId === started.session.id && tool.tool === "submit_request").length).toBe(2); + const transcript = page.getByRole("list", { name: "Conversation transcript", exact: true }); + await expect(transcript).toContainText("Verification number", { timeout: 100_000 }); + await expect.poll(async () => { + const rows = (await readFile(resolve(home, "execution.jsonl"), "utf8")).trim().split("\n").map(line => JSON.parse(line)); + return rows.some(row => row.issueId === started.session.issueId && row.event === "completed" && row.elapsedMs >= 60_000 && row.requests === 2); + }).toBe(true); + await expect.poll(async () => (await evidence()).comments.filter((row: any) => row.issueId === started.session.issueId && !row.authorAgentId).length).toBe(2); + await expect(transcript).toContainText("Your follow-up was included", { timeout: 30_000 }); + const beforeRepeat = await transcript.getByRole("listitem").count(); + await page.getByRole("button", { name: "Repeat answer", exact: true }).click(); + await expect(transcript.getByRole("listitem")).toHaveCount(beforeRepeat + 1); + await page.getByRole("button", { name: "Mute", exact: true }).click(); + expect(await page.evaluate(() => (window as any).__spekoTransport.muted)).toBe(true); + await page.getByRole("button", { name: "End call", exact: true }).click(); + await expect(page.getByRole("status").filter({ hasText: /^Call ended$/ })).toBeVisible(); + expect(await page.evaluate(() => (window as any).__spekoTransport.open)).toBe(false); + // Prove connection guidance reaches the ordinary execution provider without + // installing a template or replacing the selected agent's instructions. + await expect.poll(async () => (await evidence()).runs.find((row: any) => row.context.issueId === started.session.issueId)?.context.paperclipTaskCommunicationGuidance).toContain("Keep your existing identity, runtime, tools, and task assignment"); + const deliveredGuidance = (await evidence()).runs.find((row: any) => row.context.issueId === started.session.issueId).context.paperclipTaskCommunicationGuidance; + expect(deliveredGuidance).toContain("one focused clarification"); + expect(deliveredGuidance).toContain("durable Paperclip continuation"); + expect(deliveredGuidance).toContain("does not cancel work"); + expect(deliveredGuidance).toContain("Spoken consent does not replace governed approval"); + const foreign = await page.request.get(`${origin}/api/companies/${other.id}/voice-sessions/${started.session.id}`); + expect(foreign.status()).toBe(404); + await page.getByRole("button", { name: "Close voice panel", exact: true }).click(); + await page.goto(`/${company.issuePrefix}/apps/chat/connect?provider=speko&resume=${endpoint.id}`); + const rejoining = nextSession(); await page.getByRole("button", { name: "Start voice", exact: true }).click(); const resumed = await rejoining; + expect(resumed.session.issueId).toBe(started.session.issueId); expect(resumed.session.id).not.toBe(started.session.id); + await expect(page.getByRole("button", { name: "Mute", exact: true })).toBeEnabled(); + const journal = await page.evaluate(() => Object.entries(sessionStorage).filter(([key]) => key.startsWith("paperclip:voice-attempt:"))); + expect(journal).toHaveLength(1); + expect(journal[0][1]).not.toContain(resumed.media.transportToken); + // Reload must reconcile the exact old call before it offers a new connection. + await page.reload(); + await page.getByRole("button", { name: "Start voice", exact: true }).click(); + await expect(page.getByRole("alert").filter({ hasText: "The previous call was closed" })).toBeVisible(); + await expect.poll(async () => (await evidence()).sessions.find((row: any) => row.id === resumed.session.id)?.state).toBe("ended"); + expect((await evidence()).sessions).toHaveLength(2); + const recoveredStart = nextSession(); + await page.getByRole("button", { name: "Try again", exact: true }).click(); + const recovered = await recoveredStart; + expect(recovered.session.issueId).toBe(started.session.issueId); + await expect(page.getByRole("button", { name: "Repeat answer", exact: true })).toBeEnabled(); + await page.getByRole("button", { name: "Repeat answer", exact: true }).click(); + await expect(transcript).toContainText("Your follow-up was included"); + await page.getByRole("button", { name: "End call", exact: true }).click(); + await expect.poll(async () => (await evidence()).sessions.filter((row: any) => row.issueId === started.session.issueId).every((row: any) => row.state === "ended")).toBe(true); + await page.getByRole("button", { name: "Close voice panel", exact: true }).click(); + // Explicit New creates one new task; hanging up does not cancel accepted work. + await page.getByRole("combobox", { name: "Conversation", exact: true }).selectOption("new"); + const newStart = nextSession(); await page.getByRole("button", { name: "Start voice", exact: true }).click(); const fresh = await newStart; + expect(fresh.session.issueId).not.toBe(started.session.issueId); + await expect(page.getByRole("button", { name: "Mute", exact: true })).toBeEnabled(); + await speak("Do the benchmark; I will return after it finishes.", "after-hangup"); + const executionRows = async () => (await readFile(resolve(home, "execution.jsonl"), "utf8")).trim().split("\n").map(line => JSON.parse(line)); + await expect.poll(async () => (await executionRows()).some(row => row.issueId === fresh.session.issueId && row.event === "started")).toBe(true); + await page.getByRole("button", { name: "End call", exact: true }).click(); + await expect(page.getByRole("status").filter({ hasText: /^Call ended$/ })).toBeVisible(); + await expect.poll(async () => (await executionRows()).some(row => row.issueId === fresh.session.issueId && row.event === "completed" && row.elapsedMs >= 60_000), { timeout: 90_000 }).toBe(true); + await page.getByRole("button", { name: "Close voice panel", exact: true }).click(); + await page.getByRole("combobox", { name: "Conversation", exact: true }).selectOption("continue"); + const missedStart = nextSession(); await page.getByRole("button", { name: "Start voice", exact: true }).click(); const missed = await missedStart; + expect(missed.session.issueId).toBe(fresh.session.issueId); + await expect(transcript).toContainText("Verification number"); + expect((await evidence()).comments.filter((row: any) => row.issueId === fresh.session.issueId && !row.authorAgentId)).toHaveLength(1); + await speak("Ask the structured acceptance question before continuing.", "request-question"); + await expect.poll(() => page.evaluate(() => (window as any).__spekoTransport.question?.questions[0]?.id)).toBe("color"); + await expect(transcript).toContainText("Which color should I use?"); + const answer = () => page.evaluate(() => { + const transport = (window as any).__spekoTransport; + return transport.answer(transport.question.interactionId, [{ questionId: "color", optionIds: ["cobalt"] }], "answer-one"); + }); + const answered = await answer(); expect(answered.status).toBe("answered"); + expect(await answer()).toEqual(answered); + await expect(transcript).toContainText("Your structured answer was received", { timeout: 30_000 }); + await expect(transcript).toContainText("cobalt"); + await expect.poll(async () => (await evidence()).questionDeliveries.filter((row: any) => row.interactionId === answered.interactionId).length).toBe(1); + const questionEvidence = await evidence(); + expect(questionEvidence.interactions.find((row: any) => row.id === answered.interactionId)?.status).toBe("answered"); + expect(questionEvidence.tools.filter((row: any) => row.tool === "answer_question")).toHaveLength(1); + const delivery = questionEvidence.questionDeliveries.find((row: any) => row.interactionId === answered.interactionId); + expect(["delivered", "fallback_queued"]).toContain(delivery.status); + expect(delivery.targetRunId).toBeTruthy(); + await expect.poll(async () => (await evidence()).runs.find((row: any) => row.id === delivery.targetRunId)?.status).toBe("succeeded"); + const beforeRestart = await evidence(); + process.kill(supervisorPid, "SIGUSR2"); + // Restart has the same startup budget as the initial fixture server. + await expect.poll(async () => (await evidence()).appPid, { timeout: 120_000 }).not.toBe(beforeRestart.appPid); + await expect.poll(async () => { + try { return (await page.request.get(`${origin}/api/health`)).ok(); } catch { return false; } + }).toBe(true); + // Same session and signed tool identity cross the real process boundary. + expect(await answer()).toEqual(answered); + const beforeRecoveredRepeat = await transcript.getByRole("listitem").count(); + await page.getByRole("button", { name: "Repeat answer", exact: true }).click(); + await expect(transcript.getByRole("listitem")).toHaveCount(beforeRecoveredRepeat + 1); + await expect(transcript).toContainText("Your structured answer was received"); + expect(await page.evaluate(() => (window as any).__spekoTransport.creates)).toBe(3); + const afterRestart = await evidence(); + expect(afterRestart.sessions).toHaveLength(beforeRestart.sessions.length); + expect(afterRestart.comments).toHaveLength(beforeRestart.comments.length); + expect(afterRestart.runs).toHaveLength(beforeRestart.runs.length); + expect(afterRestart.tools.filter((row: any) => row.tool === "answer_question")).toHaveLength(1); + expect(afterRestart.questionDeliveries).toHaveLength(1); + await page.getByRole("button", { name: "End call", exact: true }).click(); + await expect.poll(async () => (await evidence()).sessions.every((row: any) => row.state === "ended")).toBe(true); + await page.getByRole("button", { name: "Close voice panel", exact: true }).click(); + const activated = await post(`/chat-endpoints/${endpoint.id}/test`, {}); + expect(activated.status).toBe("active"); + await page.goto(`/${company.issuePrefix}/apps/chat/${endpoint.id}/settings`); + await expect(page.getByRole("textbox", { name: "Your callback number", exact: true })).toBeEnabled(); + await expect(page.getByRole("button", { name: "Start voice", exact: true })).toHaveCount(0); + await expect(page.getByRole("combobox", { name: "Conversation", exact: true })).toHaveCount(0); + await expect(page.getByRole("link", {name: "Open Speko"})).toHaveAttribute("href", "https://platform.speko.ai/agents"); + await expect(page.locator('header img[src="/brands/apps/speko.svg"]').first()).toBeVisible(); + await page.getByRole("textbox", { name: "Your callback number", exact: true }).fill("+12015551234"); + await page.getByRole("checkbox", { name: "Allow this agent to call me about my tasks", exact: true }).check(); + await page.getByRole("button", { name: "Save phone setting", exact: true }).click(); + await expect(page.getByRole("status").filter({ hasText: "Callback setting saved" })).toBeVisible(); + await page.reload(); + await expect(page.getByRole("textbox", { name: "Your callback number", exact: true })).toHaveValue("+12015551234"); + await expect(page.getByRole("checkbox", { name: "Allow this agent to call me about my tasks", exact: true })).toBeChecked(); + const updatedAgent = await page.request.patch(`${origin}/api/agents/${agent.id}`, { headers: { Origin: origin }, data: { adapterConfig: { command: "fixture-phone" } } }); + expect(updatedAgent.ok()).toBe(true); + const callTask = await post(`/companies/${company.id}/issues`, { title: "Call my saved phone", status: "todo", assigneeAgentId: agent.id, responsibleUserId: "local-board" }); + await expect.poll(async () => (await executionRows()).some(row => row.event === "phone_called" && row.issueId === callTask.id), { timeout: 30_000 }).toBe(true); + const phoneExecution = (await executionRows()).find(row => row.event === "phone_called" && row.issueId === callTask.id); + expect(phoneExecution.secondRejected).toBe(true); + expect(JSON.stringify(phoneExecution.result)).not.toContain("+12015551234"); + expect(JSON.stringify(phoneExecution.result)).not.toContain("fixture-speko-key"); + expect(JSON.stringify(phoneExecution.result)).not.toContain("transportToken"); + await expect.poll(async () => (await evidence()).sessions.filter((row: any) => row.issueId === callTask.id && row.mode === "outbound_phone").length).toBe(1); + const phone = (await evidence()).sessions.find((row: any) => row.issueId === callTask.id && row.mode === "outbound_phone"); + const providerState = JSON.parse(await readFile(resolve(home, "speko-fixture-state.json"), "utf8")); + expect(providerState.sessions.filter((entry: any) => entry[1].mode === "outbound_phone")).toHaveLength(1); + const inspectedPhone = await (await page.request.get(`${origin}/api/companies/${company.id}/voice-sessions/${phone.id}`)).json(); + expect(inspectedPhone).toMatchObject({ id: phone.id, issueId: callTask.id, mode: "outbound_phone" }); + expect(JSON.stringify(inspectedPhone)).not.toContain("transportToken"); + const endedPhone = await post(`/companies/${company.id}/voice-sessions/${phone.id}/end`, {}); + expect(endedPhone.state).toBe("ended"); + const retainedTask = await (await page.request.get(`${origin}/api/issues/${callTask.id}`)).json(); + expect(retainedTask.status).not.toBe("cancelled"); + expect(retainedTask.assigneeAgentId).toBe(agent.id); + await page.getByRole("checkbox", { name: "Allow this agent to call me about my tasks", exact: true }).uncheck(); + await page.getByRole("button", { name: "Save phone setting", exact: true }).click(); + await expect(page.getByRole("status").filter({ hasText: "Callback setting saved" })).toBeVisible(); + await expect.poll(async () => (await evidence()).sessions.every((row: any) => row.state === "ended")).toBe(true); + // E08: a real signed pre-call route creates NO task before explicit live approval. + await page.getByRole("combobox", {name: "Company phone number", exact: true}).selectOption("pn_fixture"); + await page.getByRole("checkbox", {name: "Enable incoming calls on this number", exact: true}).check(); + await page.getByRole("button", {name: "Save incoming call setting", exact: true}).click(); + await expect(page.getByRole("status").filter({hasText: "Incoming call setting saved"})).toBeVisible(); + await page.request.patch(`${origin}/api/agents/${agent.id}`, {headers: {Origin: origin}, data: {adapterConfig: {command: "fixture"}}}); + const providerCommand = async (data: unknown) => { + const response = await page.request.post(`http://127.0.0.1:${port + 1}/command`, {headers: {Origin: origin}, data}); + expect(response.ok(), await response.text()).toBe(true); return response.json(); + }; + const beforeIncoming = (await evidence()).sessions.length; + const incoming = await providerCommand({action: "inbound", agentId: "agent_fixture", callerId: "+12015551234"}); + await expect(page.getByRole("heading", {name: "Incoming call awaiting approval", exact: true})).toBeVisible(); + await expect.poll(async () => (await evidence()).incoming.filter((call: any) => call.endpointId === endpoint.id).length).toBe(1); + expect((await evidence()).sessions).toHaveLength(beforeIncoming); + const unapproved = await providerCommand({sessionId: incoming.sessionId, tool: "submit_request", args: {text: "Spoofed caller ID must not grant task access"}, toolCallId: "before-approval"}); + expect(unapproved).toMatchObject({requestAccepted: false, authorization: "awaiting_approval", updates: []}); + await page.getByRole("textbox", {name: "Code spoken on your call", exact: true}).fill("000000"); + await page.getByRole("button", {name: "Approve live call", exact: true}).click(); + await expect(page.getByRole("alert").filter({hasText: "does not match"})).toBeVisible(); + const code = incoming.firstMessage.replace(/\D/g, ""); expect(code).toHaveLength(6); + await page.getByRole("textbox", {name: "Code spoken on your call", exact: true}).fill(code); + const inboundTask = await post(`/companies/${company.id}/issues`, {title: "Approved incoming phone conversation", assigneeAgentId: agent.id, status: "todo"}); + // Refresh task inventory after this fixture task is created. + await page.reload(); + await page.getByRole("textbox", {name: "Code spoken on your call", exact: true}).fill(code); + await page.getByRole("combobox", {name: "Conversation task", exact: true}).selectOption(inboundTask.id); + await page.getByRole("button", {name: "Approve live call", exact: true}).click(); + await expect(page.getByRole("heading", {name: "Call approved", exact: true})).toBeFocused(); + await expect.poll(async () => (await evidence()).sessions.find((call: any) => call.providerSessionId === incoming.sessionId)?.issueId).toBe(inboundTask.id); + const phoneWork = {sessionId: incoming.sessionId, tool: "submit_request", args: {text: "Perform this approved intake task"}, toolCallId: "approved-work"}; + const incomingAccepted = await providerCommand(phoneWork); expect(incomingAccepted).toMatchObject({status: "accepted", authorization: "approved"}); + expect(await providerCommand(phoneWork)).toEqual(incomingAccepted); + const boundIncoming = (await evidence()).sessions.find((call: any) => call.providerSessionId === incoming.sessionId); + await expect.poll(async () => (await evidence()).tools.filter((call: any) => call.sessionId === boundIncoming.id && call.tool === "submit_request").length).toBe(1); + // A saved task answer must reach the live phone transport before hangup. + // Provider acceptance is asserted separately from actual spoken playback. + await expect.poll(async () => (await evidence()).comments.some((row: any) => row.issueId === inboundTask.id && row.authorAgentId === agent.id && row.body.includes("Verification number")), {timeout: 100_000}).toBe(true); + await expect.poll(async () => (await providerCommand({action: "messages", sessionId: incoming.sessionId})).some((message: any) => message.text.includes("Verification number")), {timeout: 30_000}).toBe(true); + await expect.poll(async () => (await evidence()).pushActions.some((action: any) => action.payload.sessionId === boundIncoming.id && action.status === "completed" && action.result?.messageId && action.result.playback === "unknown")).toBe(true); + await post(`/companies/${company.id}/voice-sessions/${boundIncoming.id}/end`, {}); + // E15: report reconciliation persists one report, excludes recordings and keeps task assignment. + await expect.poll(async () => (await evidence()).reports.find((report: any) => report.sessionId === boundIncoming.id)?.status, {timeout: 45000}).toBe("available"); + const reportResponse = await page.request.get(`${origin}/api/companies/${company.id}/voice-sessions/${boundIncoming.id}/report`); + const report = await reportResponse.json(); expect(report.session.issueId).toBe(inboundTask.id); + expect(report.report).toMatchObject({costMicroUsd: "152340", transcript: [{text: "Fixture call transcript"}]}); + expect(JSON.stringify(report)).not.toContain("recording"); expect(JSON.stringify(report)).not.toContain("fixture-speko-key"); + await expect(page.getByRole("region", {name: "Your calls", exact: true})).toHaveCount(0); + await page.goto(`/${company.issuePrefix}/apps/chat/${endpoint.id}/activity`); + const history = page.getByRole("region", {name: "Connection activity", exact: true}); + await history.getByRole("button", {name: "Refresh", exact: true}).click(); + await expect(history).toContainText("Incoming call · Completed"); + await history.locator("summary").filter({hasText: "Incoming call · Completed"}).click(); + await expect(history.getByRole("list", {name: "Call transcript", exact: true})).toContainText("Fixture call transcript"); + expect((await evidence()).reports.filter((row: any) => row.sessionId === boundIncoming.id)).toHaveLength(1); + const retainedIncomingTask = await (await page.request.get(`${origin}/api/issues/${inboundTask.id}`)).json(); + expect(retainedIncomingTask.assigneeAgentId).toBe(agent.id); expect(retainedIncomingTask.status).not.toBe("cancelled"); + await page.goto(`/${company.issuePrefix}/apps/chat/${endpoint.id}/settings`); + // E09: opted-in callers start a task-scoped, low-trust conversation without sign-in. + await page.getByRole("checkbox", {name: "Let callers start new low-trust tasks without sign-in", exact: true}).check(); + await page.getByRole("button", {name: "Save incoming call setting", exact: true}).click(); + await expect(page.getByRole("status").filter({hasText: "Incoming call setting saved"})).toBeVisible(); + const guest = await providerCommand({action: "inbound", agentId: "agent_fixture", callerId: "+12015559876"}); + await expect(page.getByRole("heading", {name: "Incoming conversation", exact: true})).toBeVisible(); + const guestRequest = {sessionId: guest.sessionId, tool: "submit_request", args: {text: "Please contact me about a shipping question. Do not start private work."}, toolCallId: "guest-message"}; + const guestAccepted = await providerCommand(guestRequest); + expect(guestAccepted).toMatchObject({authorization: "guest_intake", status: "accepted"}); + expect(await providerCommand(guestRequest)).toEqual(guestAccepted); + expect(await providerCommand({sessionId: guest.sessionId, tool: "get_updates", args: {cursor: 0}, toolCallId: "guest-read"})).toMatchObject({authorization: "guest_intake", updates: []}); + await expect.poll(async () => (await evidence()).sessions.filter((call: any) => call.providerSessionId === guest.sessionId).length).toBe(1); + const guestSession = (await evidence()).sessions.find((call: any) => call.providerSessionId === guest.sessionId); + const guestTaskResponse = await page.request.get(`${origin}/api/issues/${guestSession.issueId}`); + const guestTask = await guestTaskResponse.json(); + expect(guestTask).toMatchObject({ assigneeAgentId: agent.id, responsibleUserId: null, sourceTrust: {disposition: "quarantined"}, executionPolicy: {authorizationPolicy: {trustBoundary: {allowedToolClasses: [], allowedSecretBindingIds: []}}}}); + await expect.poll(async () => (await evidence()).comments.filter((row: any) => row.issueId === guestTask.id && row.body === (guestRequest.args as {text: string}).text).length).toBe(1); + // This fixture has only Local, so containment must block execution rather + // than silently grant the public caller the selected agent's host privileges. + await expect.poll(async () => (await evidence()).runs.find((run: any) => run.context?.issueId === guestTask.id)?.errorCode).toBe("low_trust_isolation_unavailable"); + const guestRun = (await evidence()).runs.find((run: any) => run.context?.issueId === guestTask.id); + expect((await evidence()).wakeups.some((row: any) => row.runId === guestRun.id)).toBe(true); + expect((await evidence()).tools.filter((row: any) => row.sessionId === guestSession.id && row.tool === "submit_request")).toHaveLength(1); + await page.getByRole("button", {name: "End call", exact: true}).click(); + await expect(page.getByRole("heading", {name: "Caller hung up", exact: true})).toBeFocused(); + await expect.poll(async () => (await evidence()).sessions.find((call: any) => call.id === guestSession.id)?.state, {timeout: 45000}).toBe("ended"); + await expect.poll(async () => (await evidence()).reports.find((report: any) => report.sessionId === guestSession.id)?.status, {timeout: 45000}).toBe("available"); + await page.goto(`/${company.issuePrefix}/apps/chat/${endpoint.id}/activity`); + await history.getByRole("button", {name: "Refresh", exact: true}).click(); + await expect(history.locator("summary").filter({hasText: "Incoming call · Completed"})).toHaveCount(2); + expect(external).toEqual([]); + await info.attach("durable-records", { body: JSON.stringify(await evidence(), null, 2), contentType: "application/json" }); + await page.screenshot({ path: info.outputPath("completed.png"), fullPage: true }); +}); + +// Connection setup selects an existing agent without changing its runtime or +// reassigning tasks created before the connection. +test("E02: selecting an existing agent preserves its configuration and historical assignment", async ({ page }) => { + const company = await (await page.request.post("/api/companies", { data: { name: "Speko existing agent" } })).json(); + const created = await page.request.post(`/api/companies/${company.id}/agents`, { data: { name: "Existing support agent", role: "general", adapterType: "process", adapterConfig: { command: "true" }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: false } } } }); + expect(created.ok()).toBe(true); + const agent = await created.json(); + const task = await (await page.request.post(`/api/companies/${company.id}/issues`, { data: { title: "Preserve existing work", status: "backlog", assigneeAgentId: agent.id } })).json(); + await page.goto(`/${company.issuePrefix}/apps/chat/connect?provider=speko`); + await expect(page.getByRole("button", { name: "Review recommended agent" })).toHaveCount(0); + await page.getByRole("button", { name: "Choose an active agent", exact: true }).click(); + await page.getByRole("button", { name: "Select Existing support agent", exact: true }).click(); + await page.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(page.getByText(/Existing support agent keeps its existing runtime/)).toBeVisible(); + await page.reload(); + await expect(page.getByText(/Existing support agent keeps its existing runtime/)).toBeVisible(); + const agents = await (await page.request.get(`/api/companies/${company.id}/agents`)).json(); + expect(agents).toHaveLength(1); + expect(agents[0].id).toBe(agent.id); + expect(agents[0].adapterConfig).toEqual(agent.adapterConfig); + expect(agents[0].reportsTo).toBe(agent.reportsTo); + expect((await (await page.request.get(`/api/issues/${task.id}`)).json()).assigneeAgentId).toBe(agent.id); + const endpoints = await (await page.request.get(`/api/companies/${company.id}/chat-endpoints`)).json(); + expect(endpoints).toHaveLength(1); + expect(endpoints[0].assignedAgentId).toBe(agent.id); +}); + + +test("E12: insufficient credits explains funding and never blindly retries a rejected call", async ({page}) => { + const company=await (await page.request.post("/api/companies",{data:{name:"Speko credit rejection"}})).json(); + const agent=await (await page.request.post(`/api/companies/${company.id}/agents`,{data:{name:"Credit-test agent",role:"general",adapterType:"process",adapterConfig:{command:"true"},runtimeConfig:{heartbeat:{enabled:false,wakeOnDemand:false}}}})).json(); + await page.addInitScript(() => { navigator.mediaDevices.getUserMedia=async () => {throw new Error("Rejected call must not request microphone");}; }); + await page.goto(`/${company.issuePrefix}/apps/chat/connect?provider=speko`); + await page.getByRole("button",{name:"Choose an active agent",exact:true}).click(); + await page.getByRole("button",{name:"Select Credit-test agent",exact:true}).click(); + await page.getByRole("button",{name:"Continue",exact:true}).click(); + await page.getByRole("textbox",{name:"Speko agent ID",exact:true}).fill("agent_credit_fixture"); + await page.getByLabel("Speko API key",{exact:true}).fill("fixture-speko-key"); + await page.getByRole("button",{name:"Connect Speko",exact:true}).click(); + await expect(page.getByRole("button",{name:"Start voice",exact:true})).toBeVisible(); + const creditState=async () => (await page.request.post(`http://127.0.0.1:${port+1}/command`,{headers:{Origin:origin},data:{action:"credits_required",agentId:"agent_credit_fixture",enabled:true}})).json(); + expect((await creditState()).creationAttempts).toBe(0); + const responsePromise=page.waitForResponse(r => r.request().method()==="POST" && /\/voice-sessions$/.test(new URL(r.url()).pathname)); + await page.getByRole("button",{name:"Start voice",exact:true}).click(); + const response=await responsePromise;expect(response.status()).toBe(409); + const error=await response.json();expect(error.details.code).toBe("voice_credits_required"); + expect(JSON.stringify(error)).not.toContain("private fixture provider detail"); + await expect(page.getByRole("alert").filter({hasText:"Add credits in Speko"})).toBeVisible(); + await expect(page.getByRole("button",{name:"Mute",exact:true})).toHaveCount(0); + const inspected=await (await page.request.get(`/api/companies/${company.id}/voice-sessions/${error.details.sessionId}`)).json(); + expect(inspected).not.toHaveProperty("providerSessionId");expect(inspected.errorCode).toBe("insufficient_credits"); + const {home}=JSON.parse(await readFile(resolve(import.meta.dirname,".instance.json"),"utf8")); + await expect.poll(async () => JSON.parse(await readFile(resolve(home,"evidence.json"),"utf8")).sessions.find((row: {id: string}) => row.id===error.details.sessionId)?.providerSessionId).toBeNull(); + expect(["failed","ended"]).toContain(inspected.state); + // Replay the exact already-rejected creation, as after a lost response. + const replay=await page.request.post(response.url(),{data:response.request().postDataJSON()}); + expect(replay.ok()).toBe(true);expect((await replay.json()).session.id).toBe(error.details.sessionId); + expect((await creditState()).creationAttempts).toBe(1); +}); diff --git a/tests/speko-native/live-browser.mjs b/tests/speko-native/live-browser.mjs new file mode 100644 index 00000000000..3ab85384dbe --- /dev/null +++ b/tests/speko-native/live-browser.mjs @@ -0,0 +1,274 @@ +/** Explicit live acceptance; real browser UI + Speko, synthetic microphone only. + * Requires the disposable server and prior provider-persona authorization. + * No Playwright traces: setup carries a provider credential. + */ +import { chromium, expect } from '@playwright/test'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +const origin = `http://127.0.0.1:${Number(process.env.SPEKO_NATIVE_PORT ?? 3449)}`; +const privateRoot = resolve(homedir(), '.paperclip/speko-proof'); +const resumeIssueId = process.env.SPEKO_NATIVE_RESUME_ISSUE_ID; +const diskScenario = process.env.SPEKO_NATIVE_SCENARIO === 'disk'; +const delayedTemplate = process.env.SPEKO_NATIVE_SCENARIO === 'template-delayed'; +const templateScenario = diskScenario || delayedTemplate || process.env.SPEKO_NATIVE_SCENARIO === 'template'; +const expectedWord = process.env.SPEKO_NATIVE_EXPECTED_WORD ?? 'marigold'; +const evidencePath = process.env.SPEKO_NATIVE_EVIDENCE_PATH; +if (delayedTemplate && !evidencePath) throw new Error('The delayed real-agent scenario requires SPEKO_NATIVE_EVIDENCE_PATH for durable follow-up assertions'); +const fixturePath = process.env.SPEKO_NATIVE_FIXTURE_PATH ?? resolve(privateRoot, templateScenario ? 'native-template-fixture.json' : 'native-fixture.json'); +const fixture = JSON.parse(await readFile(fixturePath, 'utf8')); +const out = resolve(privateRoot, `native-browser-${Date.now()}`); +await mkdir(out, { mode: 0o700 }); +const changedFiles = execFileSync('git', ['ls-files', '-m', '-o', '--exclude-standard', '-z'], { encoding: 'utf8' }).split('\0').filter(Boolean); +const hashes = {}; +for (const file of changedFiles) { try { hashes[file] = createHash('sha256').update(await readFile(file)).digest('hex'); } catch { hashes[file] = 'deleted'; } } +await writeFile(resolve(out, 'source.json'), JSON.stringify({ commit: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), changedFiles: hashes }, null, 2), { mode: 0o600 }); +const key = (await readFile(resolve(homedir(), '.secrets'), 'utf8')).match(/^export SPEKO_MCP_API_KEY=['"]([^'"]+)['"]/m)?.[1]; +if (!key) throw new Error('Saved Speko credential missing'); +const browser = await chromium.launch({ headless: true, args: ['--autoplay-policy=no-user-gesture-required'] }); +const page = await browser.newPage({ viewport: { width: 1280, height: 960 } }); +const events = [], started = Date.now(); +const idleProbe = process.env.SPEKO_NATIVE_IDLE_PROBE === "1"; +if (idleProbe) await page.route("**/voice-sessions/*/notification", (route) => route.fulfill({ status: 200, contentType: "application/json", body: "null" })); +let session, progress; +let executionProvider = "unknown"; +const event = (kind, details = {}) => { events.push({ kind, elapsedMs: Date.now() - started, ...details }); console.log(JSON.stringify({ kind, ...details })); }; +page.on('response', async (response) => { + if (response.request().method() === 'POST' && /\/voice-sessions$/.test(new URL(response.url()).pathname) && response.ok()) { + const result = await response.json(); session = result.session; + await writeFile(resolve(out, 'session.json'), JSON.stringify(session), { mode: 0o600 }); + event('session_created', { sessionId: session.id, issueId: session.issueId }); + } +}); +await page.addInitScript(() => { + const state = window.spekoFixture = { tracks: [], events: [], context: null, destination: null, chunks: [], recorder: null, audioEdges: [], recordingStartedAt: null }; + navigator.mediaDevices.getUserMedia = async () => { + state.context ??= new AudioContext({ sampleRate: 48000 }); + await state.context.resume(); + if (!state.destination) { + state.destination = state.context.createMediaStreamDestination(); + const silence = state.context.createConstantSource(); silence.offset.value = 0.00001; silence.connect(state.destination); silence.start(); state.silence = silence; + } + const stream = state.destination.stream.clone(); state.tracks.push(...stream.getTracks()); + return stream; + }; + new MutationObserver(() => { + const audio = [...document.querySelectorAll('audio')].find((node) => node.srcObject); + if (!audio || state.recorder) return; + state.recorder = new MediaRecorder(audio.srcObject); + state.recorder.ondataavailable = (event) => { if (event.data.size) state.chunks.push(event.data); }; + state.recordingStartedAt = Date.now(); + state.recorder.start(); + // Measure received audio, independently of the model's transcript/state. + state.context ??= new AudioContext({ sampleRate: 48000 }); + const source = state.context.createMediaStreamSource(audio.srcObject); + const analyser = state.context.createAnalyser(); analyser.fftSize = 1024; + const sink = state.context.createGain(); sink.gain.value = 0; + source.connect(analyser); analyser.connect(sink); sink.connect(state.context.destination); + const samples = new Float32Array(analyser.fftSize); let audible = false; + state.audioTimer = setInterval(() => { + analyser.getFloatTimeDomainData(samples); + const rms = Math.sqrt(samples.reduce((sum, value) => sum + value * value, 0) / samples.length); + const next = rms > 0.01; + if (next) state.lastAudibleAt = Date.now(); + if (next !== audible) { state.audioEdges.push({ at: Date.now(), audible: next }); audible = next; } + }, 20); + }).observe(document, { childList: true, subtree: true }); +}); +async function speak(name) { + const audioPath = process.env.SPEKO_NATIVE_AUDIO_DIR ? resolve(process.env.SPEKO_NATIVE_AUDIO_DIR, `${name}.pcm`) : new URL(`../../.paperclip-local/speko-proof-audio/${name}.pcm`, import.meta.url); + const bytes = [...await readFile(audioPath)]; + event('input_started', { name }); + await page.evaluate(async (bytes) => { + const state = window.spekoFixture, raw = new DataView(new Uint8Array(bytes).buffer); + const buffer = state.context.createBuffer(1, bytes.length / 2, 48000); + const channel = buffer.getChannelData(0); + for (let i = 0; i < channel.length; i++) channel[i] = raw.getInt16(i * 2, true) / 32768; + const source = state.context.createBufferSource(); source.buffer = buffer; source.connect(state.destination); + await new Promise((done) => { source.onended = done; source.start(); }); source.disconnect(); + }, bytes); + event('input_ended', { name }); +} +try { + await expect.poll(async () => fetch(`${origin}/api/health`).then((response) => response.ok).catch(() => false), { timeout: 30000 }).toBe(true); + await page.goto(origin); + if (resumeIssueId) { + await page.goto(`${origin}/${fixture.companyPrefix}/issues/${resumeIssueId}`); + await page.getByRole('button', { name: `Talk to ${fixture.agentName}`, exact: true }).click(); + } else if (process.env.SPEKO_NATIVE_CONNECTED === '1') { + if (!fixture.endpointId) throw new Error('Connected qualification requires the verified endpoint fixture'); + const taskResponse = await page.request.post(`${origin}/api/companies/${fixture.companyId}/issues`, {headers: {Origin: origin}, data: {title: "Speko live browser qualification", assigneeAgentId: fixture.agentId, status: "todo"}}); + if (!taskResponse.ok()) throw new Error(`Qualification task creation rejected (${taskResponse.status()})`); + const task = await taskResponse.json(); + await page.goto(`${origin}/${fixture.companyPrefix}/issues/${task.id}`); + await page.getByRole('button', {name: `Talk to ${fixture.agentName}`, exact: true}).click(); + event('qualification_task_created', {issueId: task.id}); + event('existing_connection_used', {endpointId: fixture.endpointId}); + } else { + await page.getByRole('link', { name: 'Connectors', exact: true }).first().click(); + if (fixture.endpointId) { + // Resume the saved setup using the same UI route after an earlier failed test. + await page.goto(`${origin}/${fixture.companyPrefix}/apps/chat/connect?provider=speko&resume=${fixture.endpointId}`); + } else { + await page.locator('[data-app-slug="speko"]').getByRole('button', { name: /^(Connect Speko|Add connection Speko)$/ }).click(); + await page.getByRole('button', { name: 'Choose an active agent', exact: true }).click(); + await page.getByRole('button', { name: `Select ${fixture.agentName ?? 'Existing Paperclip Agent'}`, exact: true }).click(); + await page.getByRole('button', { name: 'Continue', exact: true }).click(); + } + await expect(page.getByLabel('Speko API key', { exact: true }).or(page.getByRole('button', { name: 'Start voice', exact: true })).filter({ visible: true }).first()).toBeVisible({ timeout: 30000 }); + if (await page.getByLabel('Speko API key', { exact: true }).isVisible()) { + await page.getByRole('textbox', { name: 'Speko agent ID', exact: true }).fill(fixture.providerAgentId); + await page.getByLabel('Speko API key', { exact: true }).fill(key); + const responsePromise = page.waitForResponse((r) => r.request().method() === 'POST' && /\/chat-endpoints\/[^/]+\/setup$/.test(new URL(r.url()).pathname), { timeout: 60000 }); + await page.getByRole('button', { name: /^(Connect|Reconnect) Speko$/ }).click(); + const response = await responsePromise; + if (!response.ok()) throw new Error(`Setup rejected (${response.status()}): ${(await response.json()).error}`); + const endpoint = await response.json(); fixture.endpointId = endpoint.id; + await writeFile(fixturePath, JSON.stringify(fixture), { mode: 0o600 }); + event('native_setup_verified', { endpointId: endpoint.id, status: endpoint.status }); + } + } + await expect(page.getByRole('button', { name: 'Start voice', exact: true })).toBeVisible({ timeout: 30000 }); + await page.screenshot({ path: resolve(out, 'before-call.png'), fullPage: true }); + await writeFile(resolve(out, 'intent.json'), JSON.stringify({ action: 'start_browser_voice', at: new Date().toISOString(), endpointId: fixture.endpointId }), { mode: 0o600 }); + if (process.env.SPEKO_NATIVE_FRESH_CONVERSATION === '1') await page.getByRole('combobox', { name: 'Conversation', exact: true }).selectOption('new'); + await page.getByRole('button', { name: 'Start voice', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Mute', exact: true })).toBeEnabled({ timeout: 30000 }); + expect(session?.assignedAgentId).toBe(fixture.agentId); + event('media_connected'); + const agentResponse = await page.request.get(`${origin}/api/agents/${session.assignedAgentId}`); + if (agentResponse.ok()) { + const agent = await agentResponse.json(); + if (typeof agent.adapterType === 'string') executionProvider = agent.adapterType; + } + event('execution_provider_inspected', {executionProvider}); + if (resumeIssueId) { expect(session?.issueId).toBe(resumeIssueId); event('existing_task_resumed', { issueId: resumeIssueId }); } + progress = setInterval(() => { void page.locator('body').innerText().then((text) => writeFile(resolve(out, 'latest-visible.txt'), text, { mode: 0o600 })).catch(() => {}); }, 3000); + await expect(page.getByRole('list', { name: 'Conversation transcript' }).getByRole('listitem')).not.toHaveCount(0, { timeout: 20000 }); + await expect(page.getByRole('status').filter({ hasText: /^Listening$/ })).toBeVisible({ timeout: 20000 }); + await page.waitForFunction(() => window.spekoFixture.lastAudibleAt && Date.now() - window.spekoFixture.lastAudibleAt > 900, undefined, { timeout: 15000 }); + await speak(diskScenario ? 'disk-request' : resumeIssueId ? 'template-resume' : delayedTemplate ? 'template-delayed-start' : templateScenario ? 'template-start' : 'start'); + await expect(page.getByRole('status').filter({ hasText: /^Speaking$/ })).toBeVisible({ timeout: 20000 }); + event('acknowledgment_speaking'); + if (evidencePath) { + await expect.poll(async () => { + const evidence = JSON.parse(await readFile(evidencePath, 'utf8')); + return evidence.tools.filter(tool => tool.sessionId === session.id && tool.tool === 'submit_request' && tool.response?.status === 'accepted').length; + }, { timeout: 20000 }).toBeGreaterThanOrEqual(1); + event('first_request_durably_accepted'); + } + if (diskScenario) await speak('disk-followup'); + if (!resumeIssueId && !diskScenario) await speak(delayedTemplate ? 'template-delayed-followup' : templateScenario ? 'template-followup' : 'followup'); + const transcript = page.getByRole('list', { name: 'Conversation transcript', exact: true }); + if (evidencePath && !resumeIssueId) { + await expect.poll(async () => { + try { + const evidence = JSON.parse(await readFile(evidencePath, 'utf8')); + return evidence.tools.filter((tool) => tool.sessionId === session.id && tool.tool === 'submit_request' && tool.response?.status === 'accepted').length; + } catch { return 0; } + }, { timeout: 30000 }).toBeGreaterThanOrEqual(2); + event('followup_durably_accepted'); + } + const agentTranscript = transcript.getByRole('listitem').filter({ has: page.locator('span').filter({ hasText: fixture.agentName ?? 'Existing Paperclip Agent' }) }); + await expect(agentTranscript).toContainText([diskScenario ? /(?:10|ten) gigabytes[\s\S]*?(?:9[.,]7|nine point seven) gigabytes/i : templateScenario ? new RegExp(expectedWord.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i') : /verification number/i], { timeout: templateScenario ? 180000 : 100000 }); + if (diskScenario) { + await expect.poll(async () => { + const response = await page.request.get(`${origin}/api/issues/${session.issueId}/comments`); + if (!response.ok()) return false; + return (await response.json()).some(comment => comment.authorAgentId === fixture.agentId + && Date.parse(comment.createdAt) >= started && /(?:disk|space|storage|filesystem)/i.test(comment.body) + && /(?:available|free)/i.test(comment.body)); + }, {timeout: 180000}).toBe(true); + event('real_agent_disk_result_persisted'); + } + if (delayedTemplate) { + await expect.poll(async () => { + const response = await page.request.get(`${origin}/api/issues/${session.issueId}/comments`); + if (!response.ok()) return false; + return (await response.json()).some((comment) => comment.authorAgentId === fixture.agentId && /silver lantern/i.test(comment.body)); + }, { timeout: 60000 }).toBe(true); + await expect(agentTranscript.filter({ hasText: new RegExp(expectedWord, "i") })).toContainText([/silver lantern/i], { timeout: 60000 }); + event('followup_applied_by_agent'); + const comments = await (await page.request.get(`${origin}/api/issues/${session.issueId}/comments`)).json(); + const runIds = [...new Set(comments.map((comment) => comment.createdByRunId).filter(Boolean))]; + const runs = await Promise.all(runIds.map(async (id) => (await page.request.get(`${origin}/api/heartbeat-runs/${id}`)).json())); + const executionMs = Math.max(...runs.filter((run) => run.status === 'succeeded').map((run) => Date.parse(run.finishedAt) - Date.parse(run.startedAt))); + expect(executionMs).toBeGreaterThanOrEqual(65000); + event('real_delayed_execution_verified', { executionMs, runIds }); + } + if (!templateScenario && evidencePath) { + const executionPath = resolve(dirname(evidencePath), 'execution.jsonl'); + await expect.poll(async () => { + const executions = (await readFile(executionPath, 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + return executions.some(run => run.event === 'completed' && run.issueId === session.issueId && run.elapsedMs >= 60_000 && run.requests >= 2); + }, { timeout: 30_000 }).toBe(true); + const executions = (await readFile(executionPath, 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + const execution = executions.find(run => run.event === 'completed' && run.issueId === session.issueId); + await expect.poll(async () => (await agentTranscript.allTextContents()).join(' ').replaceAll('*', '').includes(String(execution.number)), { timeout: 30_000 }).toBe(true); + // Wait for the queued amendment to be retrieved too. Ending as soon as the + // first answer appears would hide the very self-interruption being tested. + await expect.poll(async () => { + const evidence = JSON.parse(await readFile(evidencePath, 'utf8')); + return new Set(evidence.tools.filter(tool => tool.sessionId === session.id && tool.tool === 'get_updates').flatMap(tool => (tool.response?.updates ?? []).map(update => update.publicationId))).size; + }, { timeout: 30_000 }).toBeGreaterThanOrEqual(2); + event('delayed_execution_and_two_publications_verified', { executionMs: execution.elapsedMs, requests: execution.requests, runId: execution.runId }); + } + if (!templateScenario) await expect(transcript).toContainText(/(?:received|included)[\s\S]{0,100}(?:one|1)[\s\S]{0,60}follow.up|(?:one|1)[\s\S]{0,60}follow.up[\s\S]{0,100}(?:received|included)/i, { timeout: 30000 }); + await expect(page.getByRole('status').filter({ hasText: /^Listening$/ })).toBeVisible({ timeout: 30000 }); + await expect(agentTranscript.last()).not.toContainText(' · Speaking', { timeout: 30000 }); + await page.waitForFunction(() => window.spekoFixture.lastAudibleAt && Date.now() - window.spekoFixture.lastAudibleAt > 1800, undefined, { timeout: 20000 }); + event('delayed_result_visible'); + await page.screenshot({ path: resolve(out, 'delayed-result.png'), fullPage: true }); + await page.getByRole('button', { name: 'Mute', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Unmute', exact: true })).toHaveAttribute('aria-pressed', 'true'); + expect(await page.evaluate(() => window.spekoFixture.tracks.every((track) => !track.enabled))).toBe(true); + await page.getByRole('button', { name: 'Unmute', exact: true }).click(); + event('mute_verified'); +} catch (error) { + event('failed', { message: error.message.replaceAll(key, '[redacted]') }); + await page.screenshot({ path: resolve(out, 'failure.png'), fullPage: true }).catch(() => {}); + process.exitCode = 1; +} finally { + clearInterval(progress); + try { + const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => 'unavailable'); + await writeFile(resolve(out, 'visible.txt'), text.replaceAll(key, '[redacted]'), { mode: 0o600 }); + const audio = await page.evaluate(async () => { + const state = window.spekoFixture; + if (!state?.recorder) return null; + if (state.recorder.state !== 'inactive') await new Promise((done) => { state.recorder.onstop = done; state.recorder.stop(); }); + return [...new Uint8Array(await new Blob(state.chunks, { type: 'audio/webm' }).arrayBuffer())]; + }).catch(() => null); + const audioMeasurements = await page.evaluate(() => { + const state = window.spekoFixture; if (!state) return null; + clearInterval(state.audioTimer); + return { recordingStartedAt: state.recordingStartedAt, edges: state.audioEdges, resolutionMs: 20, rmsThreshold: 0.01 }; + }).catch(() => null); + if (audioMeasurements) await writeFile(resolve(out, 'audio-measurements.json'), JSON.stringify(audioMeasurements), { mode: 0o600 }); + if (audio) await writeFile(resolve(out, 'received.webm'), Buffer.from(audio), { mode: 0o600 }); + } catch { event('evidence_failed'); process.exitCode = 1; } finally { + // Evidence failures must never bypass hangup of a paid provider session. + if (session) { + try { + const button = page.getByRole('button', { name: /^(End call|Retry ending call)$/ }); + if (await button.count()) await button.click(); + } catch { event('ui_cleanup_unavailable'); } + try { + const response = await page.request.post(`${origin}/api/companies/${fixture.companyId}/voice-sessions/${session.id}/end`, { headers: { Origin: origin }, data: {} }); + event('end_requested', { status: response.status() }); + await expect.poll(async () => { + const inspected = await page.request.get(`${origin}/api/companies/${fixture.companyId}/voice-sessions/${session.id}`); + return inspected.ok() ? (await inspected.json()).state : 'unavailable'; + }, { timeout: 30000, intervals: [500, 1000, 2000] }).toMatch(/^(ended|failed|expired)$/); + expect(await page.evaluate(() => window.spekoFixture.tracks.every((track) => track.readyState === 'ended'))).toBe(true); + event('cleanup_verified'); + } catch { event('cleanup_unconfirmed'); process.exitCode = 1; } + } + try { + await writeFile(resolve(out, 'report.json'), JSON.stringify({ startedAt: new Date(started).toISOString(), events, session, browserNotificationDisabledForIdleProbe: idleProbe, syntheticMicrophone: true, audioReview: 'required independently; script success does not confirm complete spoken playback', resumedIssueId: resumeIssueId, diskScenario, delayedTemplate, expectedWord: templateScenario ? expectedWord : undefined, executionProvider, server: 'real', speko: 'live' }, null, 2), { mode: 0o600 }); + console.log(JSON.stringify({ evidence: out })); + } finally { await browser.close(); } + } +} diff --git a/tests/speko-native/playwright-auth.config.ts b/tests/speko-native/playwright-auth.config.ts new file mode 100644 index 00000000000..0ad530a0d5d --- /dev/null +++ b/tests/speko-native/playwright-auth.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "@playwright/test"; +import { resolve } from "node:path"; +const root = resolve(import.meta.dirname, "../.."); +const port = 3482; +export default defineConfig({ + testDir: ".", testMatch: "authenticated.spec.ts", workers: 1, retries: 0, timeout: 90_000, + outputDir: "./test-results-auth", reporter: [["list"], ["html", { outputFolder: "./playwright-report-auth", open: "never" }]], + use: { baseURL: `http://127.0.0.1:${port}`, serviceWorkers: "block", trace: "off" }, + webServer: { + command: "server/node_modules/.bin/tsx tests/speko-native/server.ts", cwd: root, + env: { ...process.env, SPEKO_NATIVE_WATCH: "0", SPEKO_NATIVE_AUTH: "1", SPEKO_NATIVE_STUB: "1", SPEKO_NATIVE_PORT: String(port), SPEKO_NATIVE_STUB_PORT: String(port + 1), SPEKO_NATIVE_CALLBACK_ORIGIN: "https://speko-fixture.invalid", SPEKO_NATIVE_INSTANCE_FILE: resolve(root, "tests/speko-native/.auth-instance.json") }, + url: `http://127.0.0.1:${port}/api/health`, reuseExistingServer: false, timeout: 120_000, + gracefulShutdown: { signal: "SIGTERM", timeout: 15_000 }, + }, +}); diff --git a/tests/speko-native/playwright.config.ts b/tests/speko-native/playwright.config.ts new file mode 100644 index 00000000000..73a75186b0a --- /dev/null +++ b/tests/speko-native/playwright.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "@playwright/test"; +import { resolve } from "node:path"; +const root = resolve(import.meta.dirname, "../.."); +const port = Number(process.env.SPEKO_NATIVE_TEST_PORT ?? 3480); +export default defineConfig({ + testDir: ".", testMatch: "integrated.spec.ts", workers: 1, fullyParallel: false, retries: 0, + timeout: 360_000, expect: { timeout: 15_000 }, + outputDir: "./test-results", reporter: [["list"], ["html", { outputFolder: "./playwright-report", open: "never" }]], + use: { serviceWorkers: "block", baseURL: `http://127.0.0.1:${port}`, viewport: { width: 1200, height: 900 }, trace: "retain-on-failure" }, + webServer: { + command: "server/node_modules/.bin/tsx tests/speko-native/server.ts", cwd: root, + env: { ...process.env, SPEKO_NATIVE_WATCH: "0", SPEKO_NATIVE_STUB: "1", SPEKO_NATIVE_PORT: String(port), SPEKO_NATIVE_STUB_PORT: String(port + 1), SPEKO_NATIVE_CALLBACK_ORIGIN: "https://speko-fixture.invalid", SPEKO_NATIVE_INSTANCE_FILE: resolve(root, "tests/speko-native/.instance.json") }, + url: `http://127.0.0.1:${port}/api/health`, reuseExistingServer: false, timeout: 120_000, + gracefulShutdown: { signal: "SIGTERM", timeout: 15_000 }, + }, +}); diff --git a/tests/speko-native/server.ts b/tests/speko-native/server.ts new file mode 100644 index 00000000000..a35e1226d18 --- /dev/null +++ b/tests/speko-native/server.ts @@ -0,0 +1,84 @@ +/** Throwaway acceptance instance: real app, DB, queues and publications. + * The execution provider is synthetic. Speko is live unless SPEKO_NATIVE_STUB=1. + * Run with server/node_modules/.bin/tsx tests/speko-native/server.ts. + */ +import { mkdtemp, writeFile, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { randomBytes } from "node:crypto"; +import { once } from "node:events"; + +const home = await mkdtemp(resolve(tmpdir(), "paperclip-speko-native-")); +const port = Number(process.env.SPEKO_NATIVE_PORT ?? 3449); +const origin = `http://127.0.0.1:${port}`; +const callbackOrigin = process.env.SPEKO_NATIVE_CALLBACK_ORIGIN; +if (!callbackOrigin || new URL(callbackOrigin).protocol !== "https:") throw new Error("Explicit HTTPS callback origin required"); +// Set isolation before importing anything that reads Paperclip configuration. +Object.assign(process.env, { + NODE_ENV: "test", PAPERCLIP_HOME: home, PAPERCLIP_INSTANCE_ID: "speko-native-acceptance", + PAPERCLIP_CONFIG: resolve(home, "config.json"), PAPERCLIP_DISABLE_CWD_ENV_FILE: "true", + PAPERCLIP_AGENT_JWT_SECRET: randomBytes(32).toString("hex"), + PAPERCLIP_DECISION_SIGNING_SECRET: randomBytes(32).toString("hex"), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomBytes(32).toString("hex"), + PAPERCLIP_SECRETS_MASTER_KEY_FILE: resolve(home, "master.key"), + PAPERCLIP_API_URL: origin, PAPERCLIP_CHAT_WEBHOOK_PUBLIC_URL: callbackOrigin, + PAPERCLIP_DEPLOYMENT_MODE: process.env.SPEKO_NATIVE_AUTH === "1" ? "authenticated" : "local_trusted", + PAPERCLIP_AUTH_PUBLIC_BASE_URL: origin, PAPERCLIP_AUTH_BASE_URL_MODE: "explicit", PAPERCLIP_DEPLOYMENT_EXPOSURE: "private", + PAPERCLIP_VITE_CACHE_DIR: resolve(home, "vite-cache"), PAPERCLIP_OPEN_ON_LISTEN: "false", + PAPERCLIP_TELEMETRY_ENABLED: "false", +}); +delete process.env.DATABASE_URL; +delete process.env.PAPERCLIP_IN_WORKTREE; +delete process.env.PAPERCLIP_MANAGED_CONFIG; +await mkdir(resolve(home, "storage")); +const { startEmbeddedPostgresTestDatabase, applyPendingMigrations } = await import("../../packages/db/src/index.js"); +const database = await startEmbeddedPostgresTestDatabase("paperclip-speko-native-db-"); +const { spawn } = await import("node:child_process"); +// The supervisor owns the temporary database. Restarting app code preserves +// calls, credentials and delivery receipts so recovery can be qualified. +const spawnApp = () => spawn("server/node_modules/.bin/tsx", [...(process.env.SPEKO_NATIVE_WATCH === "0" ? [] : ["watch"]), "tests/speko-native/app.ts"], { + cwd: resolve(import.meta.dirname, "../.."), stdio: "inherit", + env: { ...process.env, DATABASE_URL: database.connectionString }, +}); +let child = spawnApp(); +let stopping = false, restarting = false; +function watchExit() { + child.once("exit", () => { + if (stopping || restarting) return; + if (process.env.SPEKO_NATIVE_PERSIST === "1") { console.error("Live test application stopped; database retained. SIGUSR2 restarts the app."); return; } + void database.cleanup().then(() => process.exit(1)); + }); +} +watchExit(); +await writeFile(resolve(home, "instance.json"), JSON.stringify({ origin, callbackOrigin, startedAt: new Date().toISOString(), pid: process.pid }), { mode: 0o600 }); +if (process.env.SPEKO_NATIVE_INSTANCE_FILE) await writeFile(process.env.SPEKO_NATIVE_INSTANCE_FILE, JSON.stringify({ home, origin, callbackOrigin, supervisorPid: process.pid }), { mode: 0o600 }); +async function stopChild() { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill("SIGTERM"); + await exited; +} +// Only explicitly isolated fixture instances accept this restart signal. +if ((process.env.SPEKO_NATIVE_STUB === "1" || process.env.SPEKO_NATIVE_PERSIST === "1") && process.env.SPEKO_NATIVE_WATCH === "0") { + process.on("SIGUSR2", () => { + if (stopping || restarting) return; + restarting = true; + void stopChild().then(async () => { + await applyPendingMigrations(database.connectionString); + if (!stopping) { child = spawnApp(); watchExit(); } + restarting = false; + }).catch(error => { + restarting = false; + if (process.env.SPEKO_NATIVE_PERSIST === "1") console.error("Live fixture restart failed; database retained:", error.message); + else void stop(); + }); + }); +} +async function stop() { + if (stopping) return; stopping = true; + await stopChild(); + await database.cleanup(); + process.exit(0); +} +process.once("SIGINT", () => void stop()); +process.once("SIGTERM", () => void stop()); diff --git a/tests/speko-native/stub-browser-sdk.mjs b/tests/speko-native/stub-browser-sdk.mjs new file mode 100644 index 00000000000..38e921197d3 --- /dev/null +++ b/tests/speko-native/stub-browser-sdk.mjs @@ -0,0 +1,40 @@ +/** Replaces only @spekoai/client's external media boundary in browser tests. */ +export class VoiceConversation { + static async create(options) { + const state = window.__spekoTransport = { open: true, muted: false, creates: (window.__spekoTransport?.creates ?? 0) + 1, closes: window.__spekoTransport?.closes ?? 0, cursor: 0 }; + const sessionId = options.transportToken.replace('fixture:', ''); + const command = async (tool, args, toolCallId) => { + const response = await fetch('http://127.0.0.1:__STUB_PORT__/command', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ sessionId, tool, args, toolCallId }) }); + const result = await response.json(); + if (!response.ok) throw new Error(`Signed callback rejected: ${response.status}`); + return result; + }; + state.speak = async (text, toolCallId) => { + options.onMessage({ source: 'user', text, segmentId: crypto.randomUUID(), isFinal: true }); + const result = await command('submit_request', { text }, toolCallId); + options.onModeChange('speaking'); + options.onMessage({ source: 'agent', text: result.message, segmentId: crypto.randomUUID(), isFinal: true }); + options.onModeChange('listening'); + return result; + }; + state.answer = (interactionId, answers, toolCallId) => command('answer_question', { interactionId, answers }, toolCallId); + options.onModeChange('listening'); + return { + canPlaybackAudio: true, + async endSession() { if (state.open) { state.open = false; state.closes++; } }, + async setMicMuted(value) { state.muted = value; }, + async startAudioPlayback() {}, + async sendChatMessage(message) { + if (!state.open) throw new Error('Fixture media already closed'); + const result = await command('get_updates', { cursor: state.cursor, ...(message.includes('repeat=true') ? { repeat: true } : {}) }); + state.cursor = result.cursor; + for (const update of result.updates) { + if (update.question) state.question = update.question; + options.onModeChange('speaking'); + options.onMessage({ source: 'agent', text: update.text, segmentId: crypto.randomUUID(), isFinal: true }); + options.onModeChange('listening'); + } + }, + }; + } +} diff --git a/tests/speko-native/stub-provider.ts b/tests/speko-native/stub-provider.ts new file mode 100644 index 00000000000..202d0982a68 --- /dev/null +++ b/tests/speko-native/stub-provider.ts @@ -0,0 +1,130 @@ +/** Test-only external Speko transport. Paperclip routes/DB/permissions stay real. */ +import { createServer } from "node:http"; +import { createHmac, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { existsSync, readFileSync, writeFileSync, renameSync } from "node:fs"; +import { resolve } from "node:path"; + +type Tool = { id: string; name: string; source: { kind: string; url: string; secret: string }; [key: string]: unknown }; +type Session = { id: string; agentId: string; token: string; endedAt: string | null; mode: "browser" | "outbound_phone" | "inbound_phone" }; +export async function installSpekoTestProvider(origin: string, port: number) { + const originalFetch = globalThis.fetch; + const tools = new Map(), sessions = new Map(), hooks = new Map(); + const messages = new Map(); + let numberAgentId: string | null = null; + const creditRejectedAgents = new Set(), creationAttempts = new Map(); + // The simulated external provider outlives an app restart, just as Speko does. + // This file contains only deterministic fixture credentials, never a real key. + const stateFile = resolve(process.env.PAPERCLIP_HOME!, "speko-fixture-state.json"); + if (existsSync(stateFile)) { + const state = JSON.parse(readFileSync(stateFile, "utf8")); + for (const [key, value] of state.tools) tools.set(key, value); + numberAgentId = state.numberAgentId ?? null; + for (const [key, value] of state.hooks ?? []) hooks.set(key, value); + for (const [key, value] of state.messages ?? []) messages.set(key, value); + for (const [key, value] of state.sessions) sessions.set(key, value); + } + const persist = () => { + writeFileSync(`${stateFile}.pending`, JSON.stringify({ tools: [...tools], sessions: [...sessions], hooks: [...hooks], messages: [...messages], numberAgentId }), { mode: 0o600 }); + renameSync(`${stateFile}.pending`, stateFile); + }; + const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + globalThis.fetch = async (input, init) => { + const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url); + if (url.origin !== "https://api.speko.dev") return originalFetch(input, init); + if (new Headers(init?.headers).get("authorization") !== "Bearer fixture-speko-key") return json({ error: "fixture credential rejected" }, 401); + const method = init?.method ?? "GET", body = init?.body ? JSON.parse(String(init.body)) : undefined; + if (url.pathname === "/v1/phone-numbers" && method === "GET") return json([{id: "pn_fixture", organizationId: "org_fixture", e164: "+12015550123", label: "Fixture company line", agentId: numberAgentId, suspendedAt: null, direction: "both", setupStatus: {status: "ready", inboundReady: true, outboundReady: true, issues: []}}]); + if (url.pathname === "/v1/phone-numbers/pn_fixture" && method === "PATCH") {numberAgentId = body.agentId; persist(); return json({id: "pn_fixture"});} + if (url.pathname === "/v1/webhooks" && method === "GET") return json({data: [...hooks.values()].map(({signingSecret: _secret, ...hook}) => hook)}); + const hook = /^\/v1\/webhooks(?:\/([^/]+))?$/.exec(url.pathname); + if (hook && ["POST", "PATCH"].includes(method)) {const id = hook[1] ?? `hook_${randomUUID()}`; hooks.set(id, {filterTags: {}, ...body, id}); persist(); return json({id});} + const agent = /^\/v1\/agents\/([^/]+)$/.exec(url.pathname); + if (agent && method === "GET") return json({ id: agent[1], organizationId: "org_fixture", name: "Fixture voice persona" }); + if (agent && method === "PATCH") return json({ id: agent[1] }); + const tool = /^\/v1\/agents\/([^/]+)\/tools(?:\/([^/]+))?$/.exec(url.pathname); + if (tool) { + const entries = tools.get(tool[1]) ?? []; + if (method === "GET") return json(entries.map(({ source: { secret: _secret, ...source }, ...value }) => ({ ...value, source }))); + if (["POST", "PATCH"].includes(method)) { + const id = tool[2] ?? `tool_${randomUUID()}`; + tools.set(tool[1], [...entries.filter(entry => entry.id !== id), { ...body, id }]); + persist(); + return json({ id }); + } + } + if (["/v1/sessions", "/v1/sessions/phone"].includes(url.pathname) && method === "POST") { + creationAttempts.set(body.agentId, (creationAttempts.get(body.agentId) ?? 0) + 1); + if (creditRejectedAgents.has(body.agentId)) return json({error: "private fixture provider detail", code: "INSUFFICIENT_CREDITS"},402); + const id = randomUUID(); + sessions.set(id, { id, agentId: body.agentId, token: body.toolSecrets.paperclip_session_token, endedAt: null, mode: url.pathname.endsWith("/phone") ? "outbound_phone" : "browser" }); + persist(); + if (url.pathname.endsWith("/phone")) return json({ sessionId: id, status: "dialing" }); + return json({ sessionId: id, transportToken: `fixture:${id}`, transportUrl: "wss://speko-transport.invalid" }); + } + const messageCall = /^\/v1\/calls\/([^/]+)\/messages$/.exec(url.pathname); + if (messageCall && method === "POST") { + const session = sessions.get(messageCall[1]); + if (!session) return json({error: "unknown fixture call"}, 404); + if (session.endedAt) return json({error: "call already ended"}, 409); + if (typeof body.text !== "string" || !body.text.trim() || body.mode !== "respond") return json({error: "invalid message"}, 400); + const message = {id: `message_${randomUUID()}`, sessionId: session.id, text: body.text, mode: body.mode}; + messages.set(session.id, [...messages.get(session.id) ?? [], message]); persist(); + return json({message_id: message.id}, 202); + } + const call = /^\/v1\/calls\/([^/]+)(\/end)?$/.exec(url.pathname); + if (call) { + const session = sessions.get(call[1]); if (!session) return json({ error: "unknown fixture call" }, 404); + if (call[2] && method === "POST") { session.endedAt = new Date().toISOString(); persist(); return json({ status: "already_ended" }); } + if (method === "GET") return json({ id: session.id, status: session.endedAt ? "ended" : "active", endedAt: session.endedAt, ended_at: session.endedAt, duration_seconds: session.endedAt ? 60 : null, report: session.endedAt ? {session_id: session.id, cost_micro_usd: "152340", updated_at: session.endedAt, transcript: {entries: [{id: "fixture-report-turn", index: 0, source: "user", text: "Fixture call transcript", started_at: session.endedAt, ended_at: session.endedAt}]} } : null, recording_resource_uri: "private-fixture-recording-do-not-fetch" }); + } + throw new Error(`Unimplemented Speko fixture request: ${method} ${url.pathname}`); + }; + const server = createServer(async (req, res) => { + if (req.headers.origin !== origin) { res.writeHead(403).end(); return; } + res.setHeader("access-control-allow-origin", origin); + res.setHeader("access-control-allow-headers", "content-type"); + res.setHeader("access-control-allow-methods", "POST, OPTIONS"); + if (req.method === "OPTIONS") { res.writeHead(204).end(); return; } + if (req.method !== "POST" || req.url !== "/command") { res.writeHead(404).end(); return; } + try { + const chunks: Buffer[] = []; let size = 0; + for await (const chunk of req) { size += chunk.length; if (size > 64 * 1024) throw new Error("Fixture payload too large"); chunks.push(chunk); } + const command = JSON.parse(Buffer.concat(chunks).toString()); + if (command.action === "credits_required") { + if (command.enabled) creditRejectedAgents.add(command.agentId); else creditRejectedAgents.delete(command.agentId); + res.writeHead(200, {"content-type": "application/json"}).end(JSON.stringify({creationAttempts: creationAttempts.get(command.agentId) ?? 0})); return; + } + if (command.action === "inbound") { + const hook = [...hooks.values()].find(h => h.agentIds.includes(command.agentId) && h.events.includes("call.pre_call")); + if (!hook || numberAgentId !== command.agentId) {res.writeHead(409).end(); return;} + const id = randomUUID(), body = JSON.stringify({type: "call.pre_call", call_id: id, session_id: id, organization_id: "org_fixture", direction: "inbound", phone_number_id: "pn_fixture", dialed_number: "+12015550123", from: command.callerId ?? "+12015551234"}); + const timestamp = String(Math.floor(Date.now()/1000)), webhookId = `msg_${id}`; + const signature = createHmac("sha256", Buffer.from(hook.signingSecret.slice(6), "base64")).update(`${webhookId}.${timestamp}.${body}`).digest("base64"); + const response = await originalFetch(`${origin}${new URL(hook.url).pathname}`, {method: "POST", headers: {"content-type": "application/json", "webhook-id": webhookId, "webhook-timestamp": timestamp, "webhook-signature": `v1,${signature}`}, body}); + const payload = await response.json() as any; + if (!response.ok) {res.writeHead(response.status).end(JSON.stringify(payload)); return;} + sessions.set(id, {id, agentId: command.agentId, token: payload.toolSecrets.paperclip_session_token, endedAt: null, mode: "inbound_phone"}); persist(); + res.writeHead(200, {"content-type": "application/json"}).end(JSON.stringify({sessionId: id, firstMessage: payload.firstMessage})); return; + } + if (command.action === "messages") { + res.writeHead(200, {"content-type": "application/json"}).end(JSON.stringify(messages.get(command.sessionId) ?? [])); return; + } + const session = sessions.get(command.sessionId); + const tool = session && tools.get(session.agentId)?.find(item => item.name === command.tool); + if (!session || session.endedAt || !tool) { res.writeHead(404).end(); return; } + const toolId = command.toolCallId ?? randomUUID(); + const body = JSON.stringify({ session_id: session.id, tool_call_id: toolId, idempotency_key: `${session.id}:${toolId}`, tool: command.tool, args: command.args }); + const timestamp = String(Math.floor(Date.now() / 1000)), webhookId = `msg_${toolId}`; + const signature = createHmac("sha256", Buffer.from(tool.source.secret.slice(6), "base64")).update(`${webhookId}.${timestamp}.${body}`).digest("base64"); + // Only external transport is stubbed: enter the production signed route. + const response = await originalFetch(`${origin}${new URL(tool.source.url).pathname}`, { method: "POST", headers: { + "content-type": "application/json", "webhook-id": webhookId, "webhook-timestamp": timestamp, + "webhook-signature": `v1,${signature}`, authorization: `Bearer ${session.token}`, + }, body }); + res.writeHead(response.status, { "content-type": "application/json" }).end(await response.text()); + } catch { res.writeHead(500).end(JSON.stringify({ error: "Fixture command failed" })); } + }).listen(port, "127.0.0.1"); + await once(server, "listening"); + return async () => { globalThis.fetch = originalFetch; server.close(); server.closeAllConnections(); }; +} diff --git a/tests/speko-native/tsconfig.json b/tests/speko-native/tsconfig.json new file mode 100644 index 00000000000..5d2144eeb5a --- /dev/null +++ b/tests/speko-native/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "../..", + "types": ["node"], + "typeRoots": ["../../cli/node_modules/@types", "../../server/node_modules/@types"] + }, + "include": ["./**/*.ts", "../../server/src/types/express.d.ts"] +} diff --git a/tests/speko-voice-proof/.gitignore b/tests/speko-voice-proof/.gitignore new file mode 100644 index 00000000000..9e6af7f6389 --- /dev/null +++ b/tests/speko-voice-proof/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +reports/ +.env +*.pcm +*.wav diff --git a/tests/speko-voice-proof/README.md b/tests/speko-voice-proof/README.md new file mode 100644 index 00000000000..7d83f7e36ab --- /dev/null +++ b/tests/speko-voice-proof/README.md @@ -0,0 +1,149 @@ +# Speko delayed voice feasibility probe + +This isolated harness implements the first gate in the approved Speko connection +plan. It does **not** implement a Paperclip connection, touch Paperclip tasks, +or qualify browser/phone acceptance on its own. A hosted Speko agent must keep +retrieving a result for at least 60 seconds, accept an interruption and follow-up, +then speak the result without another request from the caller. + +The harness deliberately has no UI, microphone capture, database, company +credentials, private task data, or recording download. Production integration is +gated on real-provider evidence; its tests must not be reused as claims about +Paperclip permissions, persistence, or actual spoken playback. + +## Local protocol checks + +```sh +cd tests/speko-voice-proof +npm install --ignore-scripts --no-package-lock +npm test +``` + +The delay test uses an injected clock, so it checks the 60-second withholding +contract without a minute-long unit test. It cannot prove that Speko continues +polling, that its media runtime supports interruptions, or that audio plays. + +## Live setup + +1. Set `SPEKO_API_KEY` or `SPEKO_MCP_API_KEY` in the shell environment from a + private secret store. Never paste it into a command argument, fixture, trace, + screenshot, or issue. The API key stays in the provisioning process only. +2. Run `npm run init`. State defaults to + `~/.paperclip/speko-proof/state.json` (mode 0600, parent directory 0700). + `SPEKO_PROOF_STATE` can select a separate private path. Initialization refuses + to overwrite an existing run. The signing secret is generated locally. +3. Run `npm run serve`; it listens only on `127.0.0.1:3198` (override with + `SPEKO_PROOF_PORT`). It stops after 15 minutes. Only `/health` and the signed + `POST /speko-proof/tool` exist. Do not expose a Paperclip development server. +4. Configure an explicitly scoped HTTPS reverse proxy to this receiver. Verify + `POST /speko-proof/tool` with an unsigned JSON body returns 401. Set + `SPEKO_PROOF_PUBLIC_ORIGIN` to that HTTPS origin, with no path or credentials. +5. Run `npm run provision`. It creates a dedicated synthetic test persona and + two tools. It does not provision phone numbers or place calls. Partial + creation is recorded. If an API result is uncertain, it refuses blind retries; + reconcile the dedicated agent/tools in Speko before clearing the pending + marker. Use a fresh state/agent when changing the callback origin or secret. +6. Open the named **Paperclip delayed voice proof** agent in Speko. Its Browser + test runs the hosted deployed agent. Use only synthetic test speech. + +State contains secrets. Do not copy it into reports. The final `evidence.json` +contains event kinds, timings, counts, cursors, and the generated synthetic +verification phrase (never caller text). The probe +pins its first valid signed provider session and rejects other sessions; restart +the receiver between conversations. This first-delivery binding is acceptable +only for an isolated synthetic probe, **not** the production authority model. + +## Conversation and evidence + +1. Start the call. Note browser/device, provider session ID, UTC time, agent/model + configuration, and whether the provider UI says it retains the call. +2. Say “Start the test.” Measure speech-end to acknowledgment. +3. At roughly 15 seconds, interrupt with “Add a follow-up.” Confirm it is accepted + into the existing synthetic job. Do not repeatedly ask for the answer. +4. Stay on at least 75 seconds. Expect an automatically spoken, unpredictable + four-digit verification number and follow-up count 1 after the 60-second + delay. A fabricated number before 60 seconds is a failure. +5. Compare heard output to tool retrieval evidence and any provider transcript. + A `result_returned_to_tool` event is **not** a spoken-playback acknowledgment. + Record what was actually heard and how timings were measured. +6. End the call and stop the receiver with SIGINT. Preserve sanitized evidence. + Remove only the reverse-proxy route added for this probe; never reset an + existing proxy configuration. Do not download provider recordings by default. + +Repeat for phone using a designated test number and configured Speko number. +An untested transport remains blocked. If autonomous polling stops or the worker +cannot accept follow-ups during polling, record a failed feasibility gate and +prepare the questions in the qualification report before expanding the feature. + +The full implementation and acceptance matrix is in +`doc/plans/2026-09-11-speko-voice-e2e-runbook.md`. + +## Automated hosted-worker reproduction + +`npm run live` starts a **paid real Speko cascade session**, with a Node LiveKit +media client and this signed receiver. It does not run in the normal test suite, +request a microphone, dial a telephone, create Paperclip work, or implement a +replacement voice worker. Do not run `serve` at the same time: `live` owns the +receiver port. Configure the reachable HTTPS callback and dedicated test agent +first, using the same private state as `provision`. + +With a Speko credential already in the environment: + +```sh +NODE_ENV=production SPEKO_PROOF_LLM_PROVIDER=openai:gpt-4.1 npm run live +``` + +The default input is native `lk.chat` text, which isolates hosted tool execution +from speech recognition. It still receives real generated speech. Set +`SPEKO_PROOF_INPUT=audio`, `SPEKO_PROOF_START_AUDIO`, and +`SPEKO_PROOF_FOLLOWUP_AUDIO` to use synthetic speech instead. Both audio files +must be raw signed little-endian 16-bit PCM, mono, 48 kHz, saying “Start the test” +and “Add a follow-up.” This input also exercises hosted STT. It is not browser +SDK, microphone, Safari, phone, or audible interruption qualification. + +Optional settings: + +- `SPEKO_PROOF_OUTPUT`: a **new** private directory; reuse is rejected before + session creation. Defaults to a timestamped directory beside private state. +- `SPEKO_PROOF_PROMPT_FILE`: a per-session prompt override. The stricter tested + prompt is `continuous-polling.txt`; it has not passed the delayed-result gate. +- `SPEKO_PROOF_STATE` / `SPEKO_PROOF_PORT`: same meaning as the CLI. +- `SPEKO_PROOF_NOTIFY_READY=1` with + `SPEKO_PROOF_PROMPT_FILE=application-notification.txt`: send one explicitly + labeled application notification on native `lk.chat` after the synthetic + 60-second deadline. It carries no answer or authority; the agent still fetches + the result through the signed tool. This is a typed **user-role** transport, + not Speko's unsupported contextual-update channel. Product UI must preserve + application provenance rather than attribute it to something the caller said. +- `SPEKO_PROOF_NOTIFY_WEB_JOIN=1`: experimental separate, data-only participant + obtained through SDK 0.5.3's `calls.webJoin`. Requires notifications enabled. + No audio is published because web-join audio invokes agent takeover. A joined + transport does not establish that the worker will consume its text. +- `SPEKO_PROOF_INTERRUPT_ACK=1`: inject the follow-up while acknowledgment + audio is arriving, rather than after a fixed 15 seconds. Private audio-burst + timings permit inspection of when playback stopped; this does not establish + physical-device playback latency. + +Each run journals creation intent and the resulting session ID without media +credentials. It sends two instructions and detects excess submissions, waits for +the delayed result, then asks Speko to end the session. A 180-second provider cap +bounds abandoned runs; creation is never automatically retried. SIGINT/SIGTERM +request cleanup. Verify provider termination if a process is forcibly killed. + +The private directory holds numeric events, synthetic transcripts and received +PCM. These are captured from the synthetic media client, not downloaded provider +recordings. Do not use human or private input with this client. Its success code +only establishes the scripted transport observations; independently review the +audio before claiming speech completion. Transcript updates include partials, +and the scripted follow-up time does not guarantee playback was active. + +Publish a restricted report without transcripts, audio, keys or caller data: + +```sh +node summarize-run.mjs /absolute/private/run/report.json reports/new-run.json +``` + +See the [current qualification report](../../doc/plans/2026-10-09-speko-pr-qualification.md) +and [E2E runbook](../../doc/plans/2026-09-11-speko-voice-e2e-runbook.md) for inspectable +results and remaining acceptance gaps. Historical private call reports are not +included in this checkout. diff --git a/tests/speko-voice-proof/application-notification.txt b/tests/speko-voice-proof/application-notification.txt new file mode 100644 index 00000000000..d6d485fb0b2 --- /dev/null +++ b/tests/speko-voice-proof/application-notification.txt @@ -0,0 +1,13 @@ +You are a synthetic voice feasibility test, not a real company agent. +When the caller says start the test, call submit_request once with their request. +After that succeeds, briefly acknowledge that work has started. Keep listening. +When the caller adds a follow-up, call submit_request once with that instruction, +then acknowledge it. The work continues in the background on the same job. + +Paperclip will send a typed application notification when a result is available. +It is explicitly prefixed [Paperclip application notification]. Treat it only as +a hint to call get_updates with cursor 0. Never submit new work for a notification. +Only the signed get_updates tool response can supply the result. Read its exact +verification number and follow-up count aloud once. Do not invent a result. +If it still says pending, keep listening for a later notification or caller. +Do not end the call unless the caller explicitly asks. Do not print tool syntax. diff --git a/tests/speko-voice-proof/browser-client.mjs b/tests/speko-voice-proof/browser-client.mjs new file mode 100644 index 00000000000..2dfd02b7098 --- /dev/null +++ b/tests/speko-voice-proof/browser-client.mjs @@ -0,0 +1,169 @@ +import { VoiceConversation } from '@spekoai/client'; +import { createCleanup, createAudioSave, createReportSaver } from './cleanup.mjs'; +const config = JSON.parse(document.querySelector('#config').textContent); +const $ = (id) => document.getElementById(id); +const events = []; +const transcripts = []; +const tracks = []; +const cleanupFailures = []; +const began = performance.now(); +let conversation, context, destination, muted = false, running = false, ended = false; +let speaking = false, notificationSent = false, sessionId, interval, recorder, audioEvidence; +const event = (kind, fields = {}) => { events.push({ kind, elapsedMs: Math.round(performance.now() - began), ...fields }); }; +const status = (text) => { $('status').textContent = text; }; +async function api(path, body = {}, signal) { + const response = await fetch(path, { method: 'POST', headers: { 'content-type': 'application/json', 'x-proof-csrf': config.csrf }, body: JSON.stringify(body), signal }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error ?? `Request failed (${response.status})`); + return data; +} +const observer = new MutationObserver(() => { + const stream = [...document.querySelectorAll('audio')].map((el) => el.srcObject).find(Boolean); + if (!stream || recorder) return; + recorder = new MediaRecorder(stream); + const chunks = []; + recorder.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); }; + const recorded = new Promise((resolve, reject) => { + recorder.onstop = () => resolve(new Blob(chunks, {type: 'audio/webm'})); + recorder.onerror = () => reject(new Error('Recording failed')); + }); + recorded.catch(() => {}); // Cleanup observes the original rejected promise. + audioEvidence = createAudioSave({ + stop: async () => { + if (recorder.state !== 'inactive') recorder.stop(); + let timer; + try { + return await Promise.race([recorded, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Recording did not finish')), 10_000); + })]); + } finally { clearTimeout(timer); } + }, + upload: async blob => { + const response = await fetch('/audio', {method: 'POST', headers: {'content-type': 'audio/webm', 'x-proof-csrf': config.csrf}, body: blob, signal: AbortSignal.timeout(20_000)}); + if (!response.ok) throw new Error('Received audio could not be saved'); + event('received_audio_saved', {ok: true}); + }, + }); + recorder.start(); event('received_audio_recording_started'); +}); +observer.observe(document.body, { childList: true, subtree: true }); +// This fixture replaces ONLY the microphone at the browser boundary. The +// production SDK and hosted STT/LLM/TTS/transport remain real. Never call the +// original getUserMedia, including on a fallback path. +navigator.mediaDevices.getUserMedia = async () => { + if (!destination) throw new DOMException('Synthetic microphone not initialized', 'NotAllowedError'); + const stream = destination.stream.clone(); tracks.push(...stream.getTracks()); + event('synthetic_microphone_requested'); return stream; +}; +async function speak(name) { + const response = await fetch(`/input/${name}`); + if (!response.ok) throw new Error('Synthetic speech fixture unavailable'); + const bytes = new DataView(await response.arrayBuffer()); + const buffer = context.createBuffer(1, bytes.byteLength / 2, 48000); + const channel = buffer.getChannelData(0); + for (let i = 0; i < channel.length; i++) channel[i] = bytes.getInt16(i * 2, true) / 32768; + const source = context.createBufferSource(); source.buffer = buffer; source.connect(destination); + event('input_start', { label: name }); + await new Promise((resolve) => { source.onended = resolve; source.start(); }); + source.disconnect(); event('input_end', { label: name }); +} +async function waitFor(predicate, timeout, label) { + const deadline = performance.now() + timeout; + while (!predicate()) { + if (ended) throw new Error('Conversation ended'); + if (performance.now() >= deadline) throw new Error(`Timed out: ${label}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} +function renderTranscript(list) { + transcripts.splice(0, transcripts.length, ...list); + $('transcript').replaceChildren(...list.map((m) => { + const item = document.createElement('li'); item.textContent = `${m.source}: ${m.text}`; return item; + })); +} +const saveQueuedReport = createReportSaver((report, signal) => api("/report", report, signal), {timeoutMs: 10_000}); +async function saveReport() { + const report = { sessionId, sdkVersion: '0.0.13', syntheticMicrophone: true, events, transcripts, cleanupFailures, + sdkOpen: conversation?.isOpen() ?? false, audioElements: document.querySelectorAll('audio').length, + inputTrackStates: tracks.map((t) => ({ enabled: t.enabled, readyState: t.readyState })) }; + await saveQueuedReport(report); + $('evidence').textContent = JSON.stringify({ ...report, transcripts: `${transcripts.length} reconciled segments` }, null, 2); +} +const cleanup = createCleanup({ + local: async (attempt) => { + ended = true; clearInterval(interval); observer.disconnect(); + $('scenario').disabled = $('mute').disabled = true; + await attempt('recording.save', () => audioEvidence?.save()); + await attempt('sdk.end', () => conversation?.endSession()); + event('sdk_cleanup', { inputTracksEnded: tracks.every((t) => t.readyState === 'ended'), audioElements: document.querySelectorAll('audio').length }); + await attempt('tracks.stop', () => { destination?.stream.getTracks().forEach((t) => t.stop()); tracks.forEach((t) => t.stop()); }); + await attempt('context.close', () => context?.close()); + }, + provider: () => api('/end'), + report: saveReport, + failed: (operation) => { cleanupFailures.push({ operation }); event('cleanup_failed', { operation }); }, +}); +async function end() { + $('end').disabled = true; + try { + const completed = await cleanup.end(); + status(completed ? 'Ended' : 'Cleanup failed. Retry End and inspect evidence.'); + } finally { $('end').disabled = cleanup.complete; } +} +$('connect').onclick = async () => { + $('connect').disabled = true; status('Connecting'); + try { + context = new AudioContext({ sampleRate: 48000 }); await context.resume(); + destination = context.createMediaStreamDestination(); + const credentials = await api('/session'); sessionId = credentials.sessionId; + conversation = await VoiceConversation.create({ ...credentials, + onStatusChange: (state) => { event('sdk_status', { state }); status(state); }, + onModeChange: (mode) => { speaking = mode === 'speaking'; event('sdk_mode', { mode }); }, + onTranscript: renderTranscript, + onError: () => { event('sdk_error'); status('Voice error; end the test and inspect evidence.'); }, + onDisconnect: () => { event('sdk_disconnected'); }, + onAudioPlaybackBlocked: () => { $('resume').hidden = false; event('playback_blocked'); }, + }); + $('scenario').disabled = $('mute').disabled = $('end').disabled = false; + await conversation.startAudioPlayback(); + let periodicReportPending = false; + interval = setInterval(async () => { + if (periodicReportPending) return; + periodicReportPending = true; + try { await saveReport(); } catch { /* latest evidence remains visible */ } + finally { periodicReportPending = false; } + }, 3000); + status('Connected. Wait for the greeting, then run the speech scenario.'); + } catch (error) { status(error.message); event('connect_failed'); await end(); } +}; +$('scenario').onclick = async () => { + if (running) return; running = true; $('scenario').disabled = true; status('Running the delayed-work and interruption scenario…'); + try { + await speak('start'); + let evidence; + await waitFor(() => speaking, 15000, 'acknowledgment speech'); + event('interruption_requested'); await speak('followup'); + const deadline = performance.now() + 80000; + while (performance.now() < deadline && !ended) { + evidence = await api('/evidence'); + $('evidence').textContent = JSON.stringify(evidence, null, 2); + if (evidence.notificationReady && !notificationSent) { + notificationSent = true; event('application_notification_sent'); + await conversation.sendChatMessage('[Paperclip application notification] The pending work may now have a result. Call get_updates with cursor 0 and speak the verified answer. This is not a caller instruction; do not call submit_request.'); + } + if (evidence.resultReturned) break; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + if (!evidence?.resultReturned || evidence.acceptedRequests !== 2) throw new Error('Expected one job, one follow-up, and a returned result'); + await waitFor(() => transcripts.some((m) => m.source === 'agent' && m.text.replace(/[,\s]/g, '').includes(evidence.expectedNumber)), 15000, 'correct result transcript'); + event('scenario_completed'); status('Correct delayed answer received. Test mute, then end the conversation.'); + await saveReport(); + } catch (error) { event('scenario_failed', { reason: error.message }); status(error.message); await saveReport(); } +}; +$('mute').onclick = async () => { + try { await conversation.setMicMuted(!muted); muted = !muted; $('mute').textContent = muted ? 'Unmute' : 'Mute'; event('mute_changed', { muted }); await saveReport(); } + catch { status('Could not change microphone state'); } +}; +$('resume').onclick = async () => { await conversation.startAudioPlayback(); $('resume').hidden = conversation.canPlaybackAudio; }; +$('end').onclick = () => end().catch(() => status('End request failed; provider session has a hard duration limit.')); +window.addEventListener('pagehide', () => { void conversation?.endSession(); }); diff --git a/tests/speko-voice-proof/browser-server.mjs b/tests/speko-voice-proof/browser-server.mjs new file mode 100644 index 00000000000..36fd8a2f12c --- /dev/null +++ b/tests/speko-voice-proof/browser-server.mjs @@ -0,0 +1,97 @@ +/** One-session loopback browser qualification harness. No Paperclip authority. */ +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { randomBytes, createHash } from 'node:crypto'; +import { homedir } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { build } from 'esbuild'; +import { createReportSaver } from './cleanup.mjs'; +import { createProof, createProofServer, DELAY_MS } from './proof.mjs'; +const key = process.env.SPEKO_MCP_API_KEY; +if (!key) throw new Error('Credential required in environment'); +const state = JSON.parse(await readFile(`${homedir()}/.paperclip/speko-proof/state.json`, 'utf8')); +const out = resolve(process.env.SPEKO_PROOF_OUTPUT ?? `${homedir()}/.paperclip/speko-proof/browser-${Date.now()}`); +await mkdir(dirname(out), { recursive: true, mode: 0o700 }); await mkdir(out, { mode: 0o700 }); +const sourceDigests = {}; +for (const file of ['browser-server.mjs', 'browser-client.mjs', 'cleanup.mjs', 'browser.html', 'proof.mjs', 'application-notification.txt']) sourceDigests[file] = createHash('sha256').update(await readFile(new URL(file, import.meta.url))).digest('hex'); +await writeFile(resolve(out, 'source.json'), JSON.stringify({ startedAt: new Date().toISOString(), sourceDigests }), { mode: 0o600 }); +const bundle = await build({ entryPoints: [new URL('./browser-client.mjs', import.meta.url).pathname], bundle: true, write: false, platform: 'browser', format: 'esm' }); +// Disk writes stay serialized even when a browser request times out. +const saveReport = createReportSaver(async body => { + await writeFile(resolve(out, 'browser-report.json'), JSON.stringify(body, null, 2), {mode: 0o600}); + await writeFile(resolve(out, 'proof.json'), JSON.stringify(proof.evidence(), null, 2), {mode: 0o600}); +}); +const inputs = new Map(); +for (const name of ['start', 'followup']) { + const data = await readFile(new URL(`../../.paperclip-local/speko-proof-audio/${name}.pcm`, import.meta.url)); + if (!data.length || data.length % 2 || data.length > 48000 * 2 * 10) throw new Error('Invalid synthetic fixture'); + inputs.set(name, data); +} +const port = Number(process.env.SPEKO_BROWSER_PORT ?? 3199); +const origin = `http://127.0.0.1:${port}`; +const launch = randomBytes(24).toString('hex'), csrf = randomBytes(24).toString('hex'), cookie = randomBytes(24).toString('hex'); +const proof = createProof({ signingSecret: state.signingSecret, onEvent: (event) => { if (event.kind === 'request_accepted') startTime = Date.now(); } }); +const hook = createProofServer(proof); +let sessionId, attempted = false, startTime, ending, stopped = false; +async function provider(path, body) { + const r = await fetch(`https://api.speko.dev${path}`, { method: 'POST', headers: { Authorization: `Bearer ${key}`, 'content-type': 'application/json' }, body: JSON.stringify(body), signal: AbortSignal.timeout(20000) }); + if (!r.ok) throw new Error(`Provider returned ${r.status}`); + return r.json(); +} +async function endSession() { + if (!sessionId) return; + ending ??= provider(`/v1/calls/${sessionId}/end`, {}); + try { await ending; } catch (error) { ending = undefined; throw error; } +} +const server = createServer(async (req, res) => { + const reply = (code, data, type = 'application/json') => { res.writeHead(code, { 'content-type': type, 'cache-control': 'no-store', 'x-content-type-options': 'nosniff' }); res.end(type === 'application/json' ? JSON.stringify(data) : data); }; + try { + if (req.headers.host !== `127.0.0.1:${port}`) return reply(403, { error: 'Wrong host' }); + if (req.method === 'GET' && req.url === `/launch/${launch}`) { + res.setHeader('set-cookie', `proof=${cookie}; HttpOnly; SameSite=Strict; Path=/`); + res.writeHead(302, { location: '/', 'referrer-policy': 'no-referrer' }); return res.end(); + } + if (!req.headers.cookie?.split(';').some((part) => part.trim() === `proof=${cookie}`)) return reply(401, { error: 'Launch this isolated proof from its private link' }); + if (req.method === 'GET' && req.url === '/') return reply(200, (await readFile(new URL('./browser.html', import.meta.url), 'utf8')).replace('CONFIG_JSON', JSON.stringify({ csrf })), 'text/html'); + if (req.method === 'GET' && req.url === '/client.js') return reply(200, bundle.outputFiles[0].contents, 'text/javascript'); + if (req.method === 'GET' && ['/input/start', '/input/followup'].includes(req.url)) return reply(200, inputs.get(req.url.split('/').at(-1)), 'application/octet-stream'); + if (req.method !== 'POST' || req.headers.origin !== origin || req.headers['x-proof-csrf'] !== csrf) return reply(403, { error: 'Same-origin request required' }); + const limit = req.url === '/audio' ? 20 * 1024 * 1024 : 256 * 1024; + const chunks = []; let size = 0; + for await (const chunk of req) { size += chunk.length; if (size > limit) return reply(413, { error: 'Request too large' }); chunks.push(chunk); } + const raw = Buffer.concat(chunks); + if (req.url === '/audio') { await writeFile(resolve(out, 'received.webm'), raw, { mode: 0o600 }); return reply(200, { saved: true }); } + const body = JSON.parse(raw.toString('utf8')); + if (req.url === '/session') { + if (attempted) return reply(409, { error: 'One session per probe. Creation is never blindly retried.' }); + attempted = true; + await writeFile(resolve(out, 'intent.json'), JSON.stringify({ action: 'create_browser_session', at: new Date().toISOString() }), { mode: 0o600 }); + const session = await provider('/v1/sessions', { mode: 'cascade', agentId: state.agentId, ttlSeconds: 120, maxDurationSeconds: 180, constraints: { allowedProviders: { llm: ['openai:gpt-4.1'] } }, systemPrompt: await readFile(new URL('./application-notification.txt', import.meta.url), 'utf8') }); + sessionId = session.sessionId; + await writeFile(resolve(out, 'session.json'), JSON.stringify({ sessionId }), { mode: 0o600 }); + return reply(200, { sessionId, transportToken: session.transportToken, transportUrl: session.transportUrl }); + } + if (req.url === '/evidence') { + const evidence = proof.evidence(); + // A server-owned clock, bound to the first signed submission, schedules + // the synthetic completion hint; the page cannot advance it. + return reply(200, { acceptedRequests: evidence.acceptedRequests, resultReturned: evidence.resultReturned, notificationReady: startTime !== undefined && Date.now() >= startTime + DELAY_MS + 1000, + expectedNumber: evidence.expectedSyntheticResult?.match(/Verification number (\d+)/)?.[1] ?? null }); + } + if (req.url === '/report') { await saveReport(body); return reply(200, { saved: true }); } + if (req.url === '/end') { await endSession(); return reply(200, { ended: true }); } + return reply(404, { error: 'Not found' }); + } catch { return reply(502, { error: 'Probe request failed; inspect private evidence. Do not retry session creation.' }); } +}); +try { + hook.listen(3198, '127.0.0.1'); await once(hook, 'listening'); + server.listen(port, '127.0.0.1'); await once(server, 'listening'); +} catch (error) { hook.close(); server.close(); throw error; } +console.log(JSON.stringify({ url: `${origin}/launch/${launch}`, output: out })); +async function stop() { + if (stopped) return; stopped = true; + try { await endSession(); } finally { proof.close(); hook.close(); hook.closeAllConnections(); server.close(); server.closeAllConnections(); clearTimeout(cap); } +} +const cap = setTimeout(() => { void stop(); }, 270000); +process.once('SIGINT', () => { void stop(); }); process.once('SIGTERM', () => { void stop(); }); diff --git a/tests/speko-voice-proof/browser.html b/tests/speko-voice-proof/browser.html new file mode 100644 index 00000000000..48f461c1a12 --- /dev/null +++ b/tests/speko-voice-proof/browser.html @@ -0,0 +1,10 @@ + +Speko browser SDK proof + +

Speko browser SDK proof

+

Live Speko service, synthetic speech, one 60-second job. Your microphone is never opened. This diagnostic is not the Paperclip connection UI.

+

Ready

+ +

Transcript

    Evidence

    No session started.
    + + diff --git a/tests/speko-voice-proof/cleanup.mjs b/tests/speko-voice-proof/cleanup.mjs new file mode 100644 index 00000000000..b9aae102666 --- /dev/null +++ b/tests/speko-voice-proof/cleanup.mjs @@ -0,0 +1,71 @@ +/** Preserve final evidence and permit retrying an unconfirmed provider hangup. */ +export function createCleanup({ local, provider, report, failed }) { + let localEnded = false, providerEnded = false, reportSaved = false, pending; + const localSuccesses = new Set(); + const attempt = async (operation, action) => { + try { await action(); return true; } + catch { failed(operation); return false; } + }; + return { + get complete() { return localEnded && providerEnded && reportSaved; }, + end() { + if (pending) return pending; + pending = (async () => { + let ok = true; + try { + if (!localEnded) { + let localOk = true; + const completed = await attempt('local.cleanup', () => local(async (operation, action) => { + if (localSuccesses.has(operation)) return; + const succeeded = await attempt(operation, action); + if (succeeded) localSuccesses.add(operation); + else localOk = false; + })); + localEnded = completed && localOk; + ok = localEnded && ok; + } + if (!providerEnded) providerEnded = await attempt('session.end', provider); + ok = providerEnded && ok; + } finally { + reportSaved = await attempt('report.save', report); + } + return ok && reportSaved; + })().finally(() => { pending = undefined; }); + return pending; + }, + }; +} + +/** Keep the finalized recording for retry until its upload is acknowledged. */ +export function createAudioSave({stop, upload}) { + let blob, saved = false, pending; + return { + save() { + if (saved) return Promise.resolve(); + if (pending) return pending; + pending = (async () => { + blob ??= await stop(); + await upload(blob); + saved = true; + })().finally(() => { pending = undefined; }); + return pending; + }, + }; +} + +/** Older writes must finish before final cleanup evidence can be stored. */ +export function createReportSaver(write, { timeoutMs } = {}) { + let pending = Promise.resolve(); + return report => { + const snapshot = structuredClone(report); + const saved = pending.then(async () => { + if (timeoutMs === undefined) return write(snapshot); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error('Report save timed out')), timeoutMs); + try { return await write(snapshot, controller.signal); } + finally { clearTimeout(timer); } + }); + pending = saved.catch(() => {}); + return saved; + }; +} diff --git a/tests/speko-voice-proof/cleanup.test.mjs b/tests/speko-voice-proof/cleanup.test.mjs new file mode 100644 index 00000000000..24345ca2901 --- /dev/null +++ b/tests/speko-voice-proof/cleanup.test.mjs @@ -0,0 +1,149 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createCleanup, createAudioSave, createReportSaver } from './cleanup.mjs'; + +test('failed hangup preserves evidence and retries only the unconfirmed provider operation', async () => { + let locals = 0, ends = 0; + const failures = [], reports = []; + const cleanup = createCleanup({ + local: async () => { locals++; }, + provider: async () => { if (++ends === 1) throw new Error('private failure'); }, + report: async () => { reports.push([...failures]); }, + failed: (operation) => failures.push(operation), + }); + assert.equal(await cleanup.end(), false); + assert.equal(cleanup.complete, false); + assert.deepEqual(reports, [['session.end']]); + assert.equal(await cleanup.end(), true); + assert.equal(cleanup.complete, true); + assert.equal(locals, 1); + assert.equal(ends, 2); + assert.equal(reports.length, 2); +}); + +test('media failures do not prevent hangup or final evidence; concurrent cleanup coalesces', async () => { + let ends = 0, reports = 0; + const failures = []; + const cleanup = createCleanup({ + local: async (attempt) => { + await attempt('sdk.end', async () => { throw new Error('sdk failure'); }); + await attempt('tracks.stop', async () => { throw new Error('media failure'); }); + }, + provider: async () => { ends++; }, + report: async () => { reports++; }, + failed: (operation) => failures.push(operation), + }); + assert.deepEqual(await Promise.all([cleanup.end(), cleanup.end()]), [false, false]); + assert.equal(ends, 1); + assert.equal(reports, 1); + assert.deepEqual(failures, ['sdk.end', 'tracks.stop']); +}); + +test('failed report can be retried without redialing or repeating a confirmed hangup', async () => { + let ends = 0, reports = 0; + const cleanup = createCleanup({ local: async () => {}, provider: async () => { ends++; }, + report: async () => { if (++reports === 1) throw new Error('disk failure'); }, failed: () => {} }); + assert.equal(await cleanup.end(), false); + assert.equal(cleanup.complete, false); + assert.equal(await cleanup.end(), true); + assert.equal(ends, 1); + assert.equal(reports, 2); +}); + + +test('retries failed local operations and keeps End available until all media cleanup succeeds', async () => { + let tracks = 0, sdk = 0, ends = 0, reports = 0; + const cleanup = createCleanup({ + local: async attempt => { + await attempt('tracks.stop', async () => { tracks++; }); + await attempt('sdk.end', async () => { if (++sdk === 1) throw new Error('temporary failure'); }); + }, + provider: async () => { ends++; }, report: async () => { reports++; }, failed: () => {}, + }); + assert.equal(await cleanup.end(), false); + assert.equal(cleanup.complete, false); + assert.equal(await cleanup.end(), true); + assert.equal(cleanup.complete, true); + assert.deepEqual({tracks, sdk, ends, reports}, {tracks: 1, sdk: 2, ends: 1, reports: 2}); +}); + + +test('failed audio upload is retryable with the same finalized recording and blocks cleanup completion', async () => { + const blob = new Blob(['synthetic-audio']); + let stops = 0, uploads = 0, ends = 0; + const evidence = createAudioSave({stop: async () => { stops++; return blob; }, upload: async value => { + assert.equal(value, blob); if (++uploads === 1) throw new Error('HTTP failure'); + }}); + const failures = []; + const cleanup = createCleanup({local: async attempt => { await attempt('recording.save', () => evidence.save()); }, + provider: async () => { ends++; }, report: async () => {}, failed: operation => failures.push(operation)}); + assert.equal(await cleanup.end(), false); + assert.equal(cleanup.complete, false); + assert.deepEqual(failures, ['recording.save']); + assert.equal(await cleanup.end(), true); + assert.equal(cleanup.complete, true); + assert.deepEqual({stops, uploads, ends}, {stops: 1, uploads: 2, ends: 1}); +}); + +test('audio evidence waits for final recording data before upload and coalesces concurrent saves', async () => { + let finish, uploads = 0; + const stopped = new Promise(resolve => { finish = resolve; }); + const evidence = createAudioSave({stop: () => stopped, upload: async () => { uploads++; }}); + const first = evidence.save(), second = evidence.save(); + assert.equal(first, second); + await Promise.resolve(); assert.equal(uploads, 0); + finish(new Blob(['synthetic-audio'])); + await first; await evidence.save(); + assert.equal(uploads, 1); +}); + +test('final report waits for an older write and preserves its cleanup failures', async () => { + let release; + const writes = [], stored = []; + const save = createReportSaver(async report => { + writes.push(report); + if (writes.length === 1) await new Promise(resolve => { release = resolve; }); + stored.push(report); + }); + const earlier = {events: ['working'], cleanupFailures: []}; + const oldSave = save(earlier); + await Promise.resolve(); + earlier.events.push('mutated after enqueue'); + const finalSave = save({events: ['ended'], cleanupFailures: ['session.end']}); + await Promise.resolve(); + assert.equal(writes.length, 1); + release(); + await Promise.all([oldSave, finalSave]); + assert.deepEqual(stored, [{events: ['working'], cleanupFailures: []}, {events: ['ended'], cleanupFailures: ['session.end']}]); +}); + +test('a failed periodic write does not prevent storing final cleanup evidence', async () => { + let writes = 0; + const save = createReportSaver(async report => { if (++writes === 1) throw new Error('failed'); return report; }); + await assert.rejects(save({events: []})); + assert.deepEqual(await save({events: ['ended']}), {events: ['ended']}); +}); + + +test('stalled periodic transport is aborted and final evidence advances behind its late server write', async () => { + let release, calls = 0; + const stored = []; + const serverSave = createReportSaver(async report => { + if (!stored.length && report.phase === 'periodic') await new Promise(resolve => { release = resolve; }); + stored.push(report); + return report; + }); + const clientSave = createReportSaver((report, signal) => { + const response = serverSave(report); + if (++calls !== 1) { release(); return response; } + return new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), {once: true}); + response.then(resolve, reject); + }); + }, {timeoutMs: 20}); + const first = clientSave({phase: 'periodic'}); + const final = clientSave({phase: 'ended', cleanupFailures: ['session.end']}); + await assert.rejects(first, /Report save timed out/); + assert.deepEqual(await final, {phase: 'ended', cleanupFailures: ['session.end']}); + assert.deepEqual(stored, [{phase: 'periodic'}, {phase: 'ended', cleanupFailures: ['session.end']}]); +}); diff --git a/tests/speko-voice-proof/cli.mjs b/tests/speko-voice-proof/cli.mjs new file mode 100644 index 00000000000..ff1fe41a9d6 --- /dev/null +++ b/tests/speko-voice-proof/cli.mjs @@ -0,0 +1,89 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { createProof, createProofServer, PROOF_PROMPT, toolDefinitions } from "./proof.mjs"; + +// This probe is intentionally outside the app. Credentials never enter a +// browser bundle, repository fixture, or printed provider response. +const statePath = resolve(process.env.SPEKO_PROOF_STATE ?? `${homedir()}/.paperclip/speko-proof/state.json`); +const save = async (state) => writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); +async function api(path, body) { + const key = process.env.SPEKO_API_KEY ?? process.env.SPEKO_MCP_API_KEY; + if (!key) throw new Error("Set SPEKO_API_KEY or SPEKO_MCP_API_KEY in the server environment"); + const response = await fetch(`https://api.speko.dev${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(15_000), + }); + // No automatic mutation retry: an uncertain creation must be reconciled. + if (!response.ok) throw new Error(`Speko ${body ? "POST" : "GET"} ${path} returned HTTP ${response.status}`); + return response.json(); +} + +async function main() { + const command = process.argv[2]; + if (command === "init") { + await mkdir(resolve(statePath, ".."), { recursive: true, mode: 0o700 }); + const state = { signingSecret: `whsec_${randomBytes(32).toString("base64")}`, createdAt: new Date().toISOString(), toolIds: {} }; + await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + console.log("Initialized private proof state. No provider calls made."); + return; + } + const state = JSON.parse(await readFile(statePath, "utf8")); + if (command === "provision") { + const definitions = toolDefinitions(process.env.SPEKO_PROOF_PUBLIC_ORIGIN); + if (state.pendingMutation) throw new Error("A previous provider mutation has an uncertain outcome. Inspect the dedicated proof agent in Speko before clearing pendingMutation in private state."); + if (!state.agentId) { + state.pendingMutation = "create_agent"; + await save(state); + const agent = await api("/v1/agents", { + name: "Paperclip delayed voice proof", + systemPrompt: PROOF_PROMPT, + firstMessage: "This is a synthetic Paperclip test. Say start the test when you are ready.", + intent: { language: "en-US", optimizeFor: "latency" }, + runMode: "cascade", + }); + if (typeof agent.id !== "string") throw new Error("Unexpected agent response; reconcile creation in Speko"); + state.agentId = agent.id; + delete state.pendingMutation; + await save(state); + } + for (const definition of definitions) { + if (state.toolIds[definition.name]) continue; + state.pendingMutation = `create_tool:${definition.name}`; + await save(state); + const tool = await api(`/v1/agents/${encodeURIComponent(state.agentId)}/tools`, { + ...definition, source: { ...definition.source, secret: state.signingSecret }, + }); + if (typeof tool.id !== "string") throw new Error("Unexpected tool response; reconcile creation in Speko"); + state.toolIds[definition.name] = tool.id; + delete state.pendingMutation; + await save(state); + } + state.publicOrigin = process.env.SPEKO_PROOF_PUBLIC_ORIGIN; + await save(state); + console.log(JSON.stringify({ agentId: state.agentId, tools: Object.keys(state.toolIds), liveQualification: "not_established" })); + return; + } + if (command === "serve") { + const port = Number(process.env.SPEKO_PROOF_PORT ?? 3198); + if (!Number.isInteger(port) || port < 1024 || port > 65535) throw new Error("Invalid SPEKO_PROOF_PORT"); + const proof = createProof({ signingSecret: state.signingSecret, onEvent: (event) => console.log(JSON.stringify(event)) }); + const server = createProofServer(proof); + server.listen(port, "127.0.0.1", () => console.log(`Synthetic probe listening on 127.0.0.1:${port}; expires after 15 minutes`)); + const stop = async () => { + proof.close(); + server.close(); + server.closeAllConnections(); + await writeFile(resolve(statePath, "../evidence.json"), `${JSON.stringify(proof.evidence(), null, 2)}\n`, { mode: 0o600 }); + }; + const expiry = setTimeout(() => { void stop(); }, 15 * 60_000); + for (const signal of ["SIGINT", "SIGTERM"]) process.once(signal, () => { clearTimeout(expiry); void stop(); }); + return; + } + throw new Error("Usage: node cli.mjs init|provision|serve"); +} + +main().catch((error) => { console.error(error.message); process.exitCode = 1; }); diff --git a/tests/speko-voice-proof/continuous-polling.txt b/tests/speko-voice-proof/continuous-polling.txt new file mode 100644 index 00000000000..df93d58911e --- /dev/null +++ b/tests/speko-voice-proof/continuous-polling.txt @@ -0,0 +1,29 @@ +You are a synthetic voice feasibility test. Only your two real registered +functions can start the test and return its result. Use native function calls; +never print function syntax or invent tool results. + +When the user says start the test, immediately invoke submit_request. Do not +claim that work started before that function succeeds. Briefly acknowledge +successful submission, then enter the retrieval loop below IN THIS SAME TURN. + +RETRIEVAL LOOP: +1. Invoke get_updates with cursor 0. +2. If status is pending, immediately invoke get_updates with cursor 0 AGAIN. +3. Repeat step 2 until status is completed. This may require many invocations + over more than a minute. Each function waits briefly to avoid busy polling. +4. Only after completed, read the returned verification number and follow-up + count aloud once, then end your turn. + +A pending response is an instruction to keep invoking get_updates, never a +reason to finish your turn. Do not end with "I'll let you know", "let me know", +or "you can continue" while a request is pending. There is no background worker +that will remind you; if you stop invoking the function, the result is lost. +You may briefly speak acknowledgment while issuing function calls, but must +continue actual calls instead of waiting for another user utterance. + +If the user interrupts with a follow-up, invoke submit_request with their new +instruction, then resume the same retrieval loop immediately. Do not create a +different task or restart the timer. If the user says end the call, stop. + +Do not invent a verification number, a completion time, or a successful action. +Do not call get_updates after you have spoken the completed result. diff --git a/tests/speko-voice-proof/live-client.mjs b/tests/speko-voice-proof/live-client.mjs new file mode 100644 index 00000000000..da7ad19fa78 --- /dev/null +++ b/tests/speko-voice-proof/live-client.mjs @@ -0,0 +1,261 @@ +/** Real hosted Speko worker; synthetic client only, never the user's microphone. + * This is not a replacement voice server and does not qualify the browser SDK. + */ +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { homedir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { setTimeout as sleep } from "node:timers/promises"; +import { once } from "node:events"; +import { createHash } from "node:crypto"; +import { Room, RoomEvent, AudioSource, AudioFrame, AudioStream, LocalAudioTrack, TrackPublishOptions, TrackSource, TrackKind, dispose } from "@livekit/rtc-node"; +import { createMediaFailure } from "./media-failure.mjs"; +import { createProof, createProofServer, DELAY_MS } from "./proof.mjs"; + +const API = "https://api.speko.dev"; +const key = process.env.SPEKO_API_KEY ?? process.env.SPEKO_MCP_API_KEY; +if (!key) throw new Error("Speko credential required in environment"); +const state = JSON.parse(await readFile(process.env.SPEKO_PROOF_STATE ?? `${homedir()}/.paperclip/speko-proof/state.json`, "utf8")); +const mode = process.env.SPEKO_PROOF_INPUT ?? "text"; +if (!["text", "audio"].includes(mode)) throw new Error("SPEKO_PROOF_INPUT must be text or audio"); +const inputAudio = new Map(); +if (mode === "audio") { + for (const path of [process.env.SPEKO_PROOF_START_AUDIO, process.env.SPEKO_PROOF_FOLLOWUP_AUDIO]) { + if (!path) throw new Error("Both synthetic PCM input paths are required"); + const data = await readFile(path); + if (!data.length || data.length % 2 || data.length > 48_000 * 2 * 10) { + throw new Error("Each PCM input must contain at most ten seconds of 48 kHz mono s16le audio"); + } + inputAudio.set(path, data); + } +} +const provider = process.env.SPEKO_PROOF_LLM_PROVIDER ?? "openai"; +// An application-origin notification over the documented typed-turn transport. +// This deliberately does not claim a trusted system-message channel or phone +// support. The notification carries no result and grants no authority: the +// hosted agent must still retrieve the result through its signed webhook tool. +const notifyReady = process.env.SPEKO_PROOF_NOTIFY_READY === "1"; +const notifyViaJoin = process.env.SPEKO_PROOF_NOTIFY_WEB_JOIN === "1"; +const interruptAcknowledgment = process.env.SPEKO_PROOF_INTERRUPT_ACK === "1"; +if (notifyViaJoin && !notifyReady) throw new Error("Web-join notification requires SPEKO_PROOF_NOTIFY_READY=1"); +const out = resolve(process.env.SPEKO_PROOF_OUTPUT ?? `${homedir()}/.paperclip/speko-proof/automated-${Date.now()}`); +await mkdir(dirname(out), { recursive: true, mode: 0o700 }); +// A new output directory is also the mutation journal: never overwrite a prior +// session's identity and accidentally turn a rerun into an invisible redial. +await mkdir(out, { mode: 0o700 }); +const startedAt = new Date().toISOString(); +const sourceDigests = {}; +for (const name of ["live-client.mjs", "proof.mjs", "media-failure.mjs"]) { + sourceDigests[name] = createHash("sha256").update(await readFile(new URL(name, import.meta.url))).digest("hex"); +} +const began = performance.now(); +const events = []; +const audioFrames = []; +const audioBursts = []; +let lastAudio = 0; +let firstAudio; +let resultAt; +let resultAudioAt; +let sessionId; +let stopping = false; +let utterance; +let expectedRequests = 0; +let protocolFailure; +const mediaFailure = createMediaFailure(() => event("audio_input_failed")); +let mediaDisconnected = false; +let interrupted = false; +const interrupt = () => { interrupted = true; }; +process.once("SIGINT", interrupt); +process.once("SIGTERM", interrupt); +const transcript = []; +function event(kind, fields = {}) { + const item = { kind, elapsedMs: Math.round(performance.now() - began), ...fields }; + events.push(item); + console.log(JSON.stringify(item)); +} +async function api(path, body) { + const response = await fetch(`${API}${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { Authorization: `Bearer ${key}`, "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) throw new Error(`Speko HTTP ${response.status} at ${path.split("/").slice(0, 3).join("/")}`); + return response.json(); +} +const proof = createProof({ signingSecret: state.signingSecret, onEvent: (e) => { + event(`tool.${e.kind}`, e); + if (e.requestCount > expectedRequests) protocolFailure = "Provider submitted work without a matching client instruction"; + if (e.kind === "result_returned_to_tool") resultAt ??= performance.now() - began; +} }); +const server = createProofServer(proof); +const room = new Room(); +let notifierRoom; +const source = new AudioSource(48_000, 1, 100); +const consumers = []; +const audioReaders = []; +room.on(RoomEvent.TrackSubscribed, (track) => { + if (track.kind !== TrackKind.KIND_AUDIO) return; + event("remote_audio_subscribed"); + const reader = new AudioStream(track, { sampleRate: 48_000, numChannels: 1 }).getReader(); + audioReaders.push(reader); + consumers.push((async () => { + while (!stopping) { + const { value: frame, done } = await reader.read(); + if (done || stopping) break; + const at = performance.now() - began; + const data = Buffer.from(new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength)); + audioFrames.push({ at, data }); + let peak = 0; + for (const value of frame.data) peak = Math.max(peak, Math.abs(value)); + if (peak > 200) { + firstAudio ??= at; + if (!audioBursts.length || at - lastAudio > 500) { + audioBursts.push({ startMs: at, lastAudibleMs: at }); + event("audio_activity_started"); + } + audioBursts.at(-1).lastAudibleMs = at; + if (resultAt !== undefined && at >= resultAt) resultAudioAt ??= at; + lastAudio = at; + } + } + })().catch(() => event("audio_stream_closed"))); +}); +room.on(RoomEvent.Disconnected, () => { mediaDisconnected = true; event("media_disconnected"); }); +room.registerTextStreamHandler("lk.transcription", async (reader, participant) => { + const text = await reader.readAll(); + transcript.push({ at: performance.now() - began, source: participant.identity === room.localParticipant?.identity ? "client" : "agent", text }); + event("transcript_received", { characters: text.length, verification: /verification/i.test(text), toolMarkup: /= end) throw new Error(`Timed out waiting for ${label}`); + await sleep(100); + } +} +async function input(text, file, label) { + expectedRequests += 1; + event("input_start", { label, mode }); + if (mode === "text") await room.localParticipant.sendText(text, { topic: "lk.chat" }); + else { + const data = inputAudio.get(file); + await mediaFailure.waitFor(() => new Promise((done) => { utterance = { data, offset: 0, done }; })); + } + event("input_end", { label }); +} +let pump; +let failure; +try { + server.listen(Number(process.env.SPEKO_PROOF_PORT ?? 3198), "127.0.0.1"); + await once(server, "listening"); + // Write intent before external mutation. This program never retries creation. + const request = { mode: "cascade", agentId: state.agentId, ttlSeconds: 120, maxDurationSeconds: 180, constraints: { allowedProviders: { llm: [provider] } } }; + if (process.env.SPEKO_PROOF_PROMPT_FILE) request.systemPrompt = await readFile(process.env.SPEKO_PROOF_PROMPT_FILE, "utf8"); + await writeFile(resolve(out, "request.json"), JSON.stringify(request, null, 2), { mode: 0o600 }); + const session = await api("/v1/sessions", request); + sessionId = session.sessionId; + await writeFile(resolve(out, "session.json"), JSON.stringify({ sessionId }, null, 2), { mode: 0o600 }); + event("session_created", { sessionId, provider, mode }); + await room.connect(session.transportUrl, session.transportToken, { autoSubscribe: true }); + const track = LocalAudioTrack.createAudioTrack("synthetic-input", source); + const options = new TrackPublishOptions(); + options.source = TrackSource.SOURCE_MICROPHONE; + await room.localParticipant.publishTrack(track, options); + pump = mediaFailure.observe((async () => { + while (!stopping) { + const frame = new Int16Array(960); + let finished; + if (utterance) { + const count = Math.min(1920, utterance.data.length - utterance.offset); + for (let i = 0; i < count / 2; i++) frame[i] = utterance.data.readInt16LE(utterance.offset + i * 2); + utterance.offset += count; + if (utterance.offset >= utterance.data.length) { finished = utterance.done; utterance = undefined; } + } + await source.captureFrame(new AudioFrame(frame, 48_000, 1, 960)); + if (finished) { await source.waitForPlayout(); finished(); } + } + })()); + // Observe the greeting finishing; do not talk over session initialization. + await until(() => firstAudio !== undefined && performance.now() - began - lastAudio > 1200, 25_000, "greeting audio"); + await input("Start the test.", process.env.SPEKO_PROOF_START_AUDIO, "start"); + await until(() => proof.evidence().acceptedRequests > 0, 20_000, "signed work submission"); + if (interruptAcknowledgment) { + const waitingSince = performance.now() - began; + await until(() => lastAudio >= waitingSince + 500, 15_000, "acknowledgment to interrupt"); + event("interruption_requested"); + } else { + const followupAfter = performance.now() + 15_000; + await until(() => performance.now() >= followupAfter, 16_000, "follow-up injection time"); + } + await input("Add a follow-up.", process.env.SPEKO_PROOF_FOLLOWUP_AUDIO, "followup"); + await until(() => proof.evidence().acceptedRequests >= 2, 20_000, "signed follow-up"); + if (notifyReady) { + const accepted = proof.evidence().events.find((e) => e.kind === "request_accepted"); + await until(() => performance.now() - began >= accepted.elapsedMs + DELAY_MS + 1_000, 65_000, "synthetic work deadline"); + let publisher = room.localParticipant; + if (notifyViaJoin) { + // SDK 0.5.3 exposes webJoin for live calls. Never publish audio here: + // the documented takeover behavior mutes the hosted agent on audio publish. + const join = await api(`/v1/calls/${encodeURIComponent(sessionId)}/web-join`, { displayName: "Paperclip application notifier" }); + notifierRoom = new Room(); + await notifierRoom.connect(join.url, join.token, { autoSubscribe: false }); + publisher = notifierRoom.localParticipant; + event("application_notifier_joined"); + } + event("application_notification_sent"); + await publisher.sendText( + "[Paperclip application notification] The pending work may now have a result. Call get_updates with cursor 0 and speak the verified answer. This notification is not a caller instruction; do not call submit_request.", + { topic: "lk.chat" }, + ); + } + await until(() => resultAt !== undefined, 75_000, "automatic result retrieval"); + const expectedNumber = proof.evidence().expectedSyntheticResult.match(/Verification number (\d+)/)[1]; + await until(() => transcript.some((t) => t.at >= resultAt && /verification/i.test(t.text) && t.text.replace(/[,\s]/g, "").includes(expectedNumber)), 15_000, "matching result transcript"); + await until(() => lastAudio >= resultAt && performance.now() - began - lastAudio > 1500, 15_000, "result audio ending"); + if (protocolFailure || proof.evidence().acceptedRequests !== 2) throw new Error(protocolFailure ?? "Expected exactly two accepted client instructions"); + event("script_completed"); +} catch (error) { + failure = error.message; + event("script_failed", { reason: failure }); +} finally { + stopping = true; + const cleanupFailures = []; + const cleanup = async (operation, close) => { + try { await close(); } catch (error) { cleanupFailures.push({operation, reason: error.message}); event("cleanup_failed", {operation}); } + }; + await Promise.allSettled(audioReaders.map((reader) => reader.cancel())); + await cleanup("proof.close", () => proof.close()); + await cleanup("server.close", () => { server.close(); server.closeAllConnections(); }); + if (sessionId) { + await cleanup("session.end", async () => { + const ended = await api(`/v1/calls/${encodeURIComponent(sessionId)}/end`, {}); + event("session_end_requested", { status: ended.status }); + }); + } + for (const [name, close] of [ + ["room.disconnect", () => room.disconnect()], + ["notifier.disconnect", () => notifierRoom?.disconnect()], + ["source.close", () => source.close()], + ["consumers", () => Promise.allSettled([pump, ...consumers])], + ["dispose", () => dispose()], + ]) { + await cleanup(name, close); + } + process.removeListener("SIGINT", interrupt); + process.removeListener("SIGTERM", interrupt); + // These files contain only this synthetic client/session, never human audio. + try { + await writeFile(resolve(out, "output.pcm"), Buffer.concat(audioFrames.map((f) => f.data)), {mode: 0o600}); + if (resultAt !== undefined) await writeFile(resolve(out, "result.pcm"), Buffer.concat(audioFrames.filter((f) => f.at >= resultAt - 250).map((f) => f.data)), {mode: 0o600}); + } catch (error) { cleanupFailures.push({operation: "audio_evidence", reason: error.message}); } + failure ??= mediaFailure.error?.message; + const report = { cleanupFailures, startedAt, sourceDigests, sessionId, mode, provider, notifyReady, notifyViaJoin, interruptAcknowledgment, failure: failure ?? null, firstAudio, resultAt, resultAudioAt, lastAudio, audioBursts, audioFrames: audioFrames.length, proof: proof.evidence(), events, syntheticTranscript: transcript, qualification: "requires_audio_and_transport_review" }; + await writeFile(resolve(out, "report.json"), JSON.stringify(report, null, 2), { mode: 0o600 }); + console.log(JSON.stringify({ reportPath: resolve(out, "report.json"), resultReturned: report.proof.resultReturned, acceptedRequests: report.proof.acceptedRequests, failure: report.failure })); + process.exitCode = failure || cleanupFailures.length ? 1 : 0; +} diff --git a/tests/speko-voice-proof/media-failure.mjs b/tests/speko-voice-proof/media-failure.mjs new file mode 100644 index 00000000000..b5bc506b783 --- /dev/null +++ b/tests/speko-voice-proof/media-failure.mjs @@ -0,0 +1,17 @@ +/** Observe media rejection immediately and release any pending scenario wait. */ +export function createMediaFailure(onFailure) { + let error, reject; + const failure = new Promise((_, fail) => { reject = fail; }); + failure.catch(() => {}); // A failure can occur between scenario waits. + return { + get error() { return error; }, + observe(pump) { + return pump.catch(cause => { + error ??= cause; + reject(error); + onFailure(error); + }); + }, + waitFor(action) { return Promise.race([Promise.resolve().then(action), failure]); }, + }; +} diff --git a/tests/speko-voice-proof/media-failure.test.mjs b/tests/speko-voice-proof/media-failure.test.mjs new file mode 100644 index 00000000000..20b122a0073 --- /dev/null +++ b/tests/speko-voice-proof/media-failure.test.mjs @@ -0,0 +1,27 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createMediaFailure} from './media-failure.mjs'; + +for (const operation of ['captureFrame', 'waitForPlayout']) { + test(`${operation} failure releases pending input and runs hangup and evidence cleanup`, async () => { + let fail, hungUp = false, reportSaved = false, observed; + const media = createMediaFailure(error => { observed = error; }); + const sourceFailure = new Error(`${operation} failed`); + const source = {captureFrame: async () => {}, waitForPlayout: async () => {}}; + source[operation] = () => new Promise((_, reject) => { fail = reject; }); + const pump = media.observe((async () => { await source.captureFrame(); await source.waitForPlayout(); })()); + const input = media.waitFor(() => new Promise(() => {})); + while (!fail) await Promise.resolve(); + fail(sourceFailure); + let failure; + try { await input; } + catch (error) { failure = error; } + finally { hungUp = true; reportSaved = true; } + await pump; + assert.equal(failure, sourceFailure); + assert.equal(observed, sourceFailure); + assert.equal(media.error, sourceFailure); + assert.equal(hungUp && reportSaved, true); + await assert.rejects(media.waitFor(() => new Promise(() => {})), sourceFailure); + }); +} diff --git a/tests/speko-voice-proof/package.json b/tests/speko-voice-proof/package.json new file mode 100644 index 00000000000..636d2db3c6c --- /dev/null +++ b/tests/speko-voice-proof/package.json @@ -0,0 +1,23 @@ +{ + "name": "@paperclipai/speko-voice-proof", + "private": true, + "type": "module", + "scripts": { + "test": "node --test *.test.mjs", + "init": "node cli.mjs init", + "serve": "node cli.mjs serve", + "provision": "node cli.mjs provision", + "live": "node live-client.mjs" + }, + "engines": { + "node": ">=24.11.0" + }, + "dependencies": { + "standardwebhooks": "1.0.0" + }, + "devDependencies": { + "@livekit/rtc-node": "1.0.0", + "@spekoai/client": "0.0.13", + "esbuild": "0.25.12" + } +} diff --git a/tests/speko-voice-proof/proof.mjs b/tests/speko-voice-proof/proof.mjs new file mode 100644 index 00000000000..358bd6c2bfa --- /dev/null +++ b/tests/speko-voice-proof/proof.mjs @@ -0,0 +1,169 @@ +/** + * Provider feasibility probe only. No Paperclip access, real tasks, or production + * credentials. A synthetic result is withheld for 60 seconds to test whether the + * hosted voice worker keeps fetching it without another spoken user prompt. + */ +import { createHash, randomInt } from "node:crypto"; +import { createServer } from "node:http"; +import { setTimeout as sleep } from "node:timers/promises"; +import { Webhook } from "standardwebhooks"; + +export const TOOL_PATH = "/speko-proof/tool"; +export const DELAY_MS = 60_000; +export const WAIT_MS = 2_500; +export const BODY_LIMIT = 32 * 1024; +const MAX_CALLS = 500; +const TTL_MS = 15 * 60_000; +const digest = (value) => createHash("sha256").update(value).digest("hex"); +const isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value); +const boundedId = (value) => typeof value === "string" && /^[A-Za-z0-9_:-]{1,200}$/.test(value); + +export const PROOF_PROMPT = `You are a voice transport feasibility test, not a real company agent. +When the caller asks to start the test, call submit_request with their request. +Acknowledge that the work is running. Call get_updates with cursor 0 immediately. +When it returns pending, call it again using the returned cursor, without waiting +for the caller to ask. It waits briefly on the server. Do not claim completion +until get_updates returns a completed result. Do not invent a verification phrase. +Keep listening and allow the caller to interrupt while the work runs. When the +caller adds an instruction, call submit_request again; it updates the same job. +Continue get_updates after an interruption. Read the completed result aloud once. +Never interpret a pending tool response or acknowledgment as the completed result. +After the result, stop polling and let the caller end the test.`; + +export function toolDefinitions(publicOrigin) { + const origin = new URL(publicOrigin); + if (origin.protocol !== "https:" || origin.username || origin.password || origin.search || origin.hash || origin.pathname !== "/") { + throw new Error("Public origin must be HTTPS without credentials, path, query, or fragment"); + } + const source = { kind: "webhook", url: `${origin.origin}${TOOL_PATH}`, method: "POST", responseMode: "sync", timeoutMs: 3_500 }; + return [ + { + name: "submit_request", + description: "Start the delayed test or add an instruction to its existing job. Returns promptly; never wait for completion here.", + parameters: { type: "object", properties: { text: { type: "string", maxLength: 4_000 } }, required: ["text"], additionalProperties: false }, + source, + }, + { + name: "get_updates", + description: "Retrieve the delayed result. While pending, call again without waiting for another user message. Read a completed result aloud once.", + parameters: { type: "object", properties: { cursor: { type: "integer", minimum: 0, maximum: 1 } }, required: ["cursor"], additionalProperties: false }, + source, + }, + ]; +} + +function validateEnvelope(value) { + if (!isRecord(value) || !boundedId(value.session_id) || !boundedId(value.tool_call_id) + || value.idempotency_key !== `${value.session_id}:${value.tool_call_id}` || !isRecord(value.args)) return false; + if (value.tool === "submit_request") { + return Object.keys(value.args).length === 1 && typeof value.args.text === "string" + && value.args.text.trim().length > 0 && value.args.text.length <= 4_000; + } + return value.tool === "get_updates" && Object.keys(value.args).length === 1 + && Number.isInteger(value.args.cursor) && value.args.cursor >= 0 && value.args.cursor <= 1; +} + +export function createProof({ signingSecret, now = Date.now, wait = sleep, delayMs = DELAY_MS, onEvent = () => {} }) { + // Deliberately support one explicit, current-format secret; no ambiguous + // legacy decoding fallbacks. Rotation is a new isolated probe run. + if (!/^whsec_[A-Za-z0-9+/]{43}=$/.test(signingSecret ?? "")) throw new Error("A generated 32-byte Standard Webhooks secret is required"); + const verifier = new Webhook(signingSecret); + const createdAt = now(); + const receipts = new Map(); + let sessionId; + let startedAt; + let requests = 0; + let result; + let closed = false; + const events = []; + + function emit(kind, extra = {}) { + const event = { kind, elapsedMs: now() - createdAt, ...extra }; + events.push(event); + onEvent(event); + } + + async function execute(envelope) { + if (envelope.tool === "submit_request") { + if (result) return { status: 409, body: { error: "proof_already_completed" } }; + const first = startedAt === undefined; + if (first) startedAt = now(); + requests += 1; + // Never retain the caller's text, even in the local evidence file. + emit(first ? "request_accepted" : "followup_accepted", { requestCount: requests }); + return { status: 200, body: { status: "pending", cursor: 0, requestCount: requests, message: "The test is running. Retrieve updates while we continue talking." } }; + } + if (startedAt === undefined) { + emit("tool_rejected", { tool: "get_updates", reason: "start_request_first" }); + return { status: 409, body: { error: "start_request_first" } }; + } + if (envelope.args.cursor === 1 && !result) return { status: 400, body: { error: "cursor_ahead_of_result" } }; + const remaining = startedAt + delayMs - now(); + if (!result && remaining > 0) await wait(Math.min(WAIT_MS, remaining)); + if (closed || now() - createdAt >= TTL_MS) return { status: 410, body: { error: "proof_expired" } }; + if (!result && now() >= startedAt + delayMs) { + result = { cursor: 1, text: `Paperclip proof complete. Verification number ${randomInt(1_000, 10_000)}. Follow-up count ${requests - 1}.` }; + emit("result_ready", { requestCount: requests, jobElapsedMs: now() - startedAt }); + } + const updates = result && envelope.args.cursor < result.cursor ? [result] : []; + emit(updates.length ? "result_returned_to_tool" : "poll_returned", { cursor: result?.cursor ?? 0 }); + return { status: 200, body: { status: result ? "completed" : "pending", cursor: result?.cursor ?? 0, updates } }; + } + + return { + async handle(rawBody, headers) { + if (Buffer.byteLength(rawBody) > BODY_LIMIT) return { status: 413, body: { error: "body_too_large" } }; + let envelope; + try { envelope = verifier.verify(rawBody, headers); } + catch { return { status: 401, body: { error: "invalid_signature" } }; } + if (!validateEnvelope(envelope)) return { status: 400, body: { error: "invalid_envelope" } }; + if (closed || now() - createdAt >= TTL_MS) return { status: 410, body: { error: "proof_expired" } }; + // Only synthetic content exists here. First signed delivery binds this + // single-run harness; a production connector must use server-owned sessions. + if (sessionId && sessionId !== envelope.session_id) return { status: 403, body: { error: "different_session" } }; + const fingerprint = digest(rawBody); + const prior = receipts.get(envelope.idempotency_key); + if (prior) return prior.fingerprint === fingerprint ? prior.response : { status: 409, body: { error: "conflicting_retry" } }; + if (receipts.size >= MAX_CALLS) return { status: 429, body: { error: "proof_call_limit" } }; + sessionId ??= envelope.session_id; + // Reserve before execute yields, so concurrent retries share one result. + const response = Promise.resolve().then(() => execute(envelope)); + receipts.set(envelope.idempotency_key, { fingerprint, response }); + return response; + }, + close() { closed = true; }, + evidence() { + return { mode: "synthetic_provider_probe", liveQualification: "not_established", acceptedRequests: requests, expectedSyntheticResult: result?.text ?? null, resultReturned: events.some((event) => event.kind === "result_returned_to_tool"), events: structuredClone(events) }; + }, + }; +} + +export function createProofServer(proof) { + const server = createServer(async (req, res) => { + const reply = (status, body) => { + res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + res.end(JSON.stringify(body)); + }; + if (req.method === "GET" && req.url === "/health") return reply(200, { mode: "synthetic_provider_probe" }); + if (req.method !== "POST" || req.url !== TOOL_PATH) return reply(404, { error: "not_found" }); + if (req.headers["content-type"]?.split(";")[0].trim() !== "application/json") return reply(415, { error: "json_required" }); + if (req.headers["content-encoding"] && req.headers["content-encoding"] !== "identity") return reply(415, { error: "encoding_not_supported" }); + const chunks = []; + let length = 0; + try { + for await (const chunk of req) { + length += chunk.length; + if (length > BODY_LIMIT) { reply(413, { error: "body_too_large" }); return; } + chunks.push(chunk); + } + const result = await proof.handle(Buffer.concat(chunks).toString("utf8"), req.headers); + reply(result.status, result.body); + } catch { + if (!res.headersSent) reply(500, { error: "proof_failed" }); + } + }); + server.requestTimeout = 5_000; + server.headersTimeout = 5_000; + server.maxConnections = 16; + return server; +} diff --git a/tests/speko-voice-proof/proof.test.mjs b/tests/speko-voice-proof/proof.test.mjs new file mode 100644 index 00000000000..df95544573f --- /dev/null +++ b/tests/speko-voice-proof/proof.test.mjs @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { once } from "node:events"; +import test from "node:test"; +import { Webhook } from "standardwebhooks"; +import { BODY_LIMIT, createProof, createProofServer, TOOL_PATH, toolDefinitions, WAIT_MS } from "./proof.mjs"; + +const secret = () => `whsec_${randomBytes(32).toString("base64")}`; +function fixture(options = {}) { + const signingSecret = secret(); + const proof = createProof({ signingSecret, ...options }); + const signer = new Webhook(signingSecret); + let sequence = 0; + function request(tool, args, { session = "session_1", call = `call_${++sequence}`, timestamp = new Date(), signing = signer } = {}) { + const body = JSON.stringify({ tool, args, session_id: session, tool_call_id: call, idempotency_key: `${session}:${call}` }); + const id = `${session}:${call}`; + const headers = { "webhook-id": id, "webhook-timestamp": String(Math.floor(timestamp.getTime() / 1000)), "webhook-signature": signing.sign(id, timestamp, body) }; + return { body, headers, send: () => proof.handle(body, headers) }; + } + return { proof, request }; +} + +test("requires HTTPS and keeps each fetch below the four-second provider limit", () => { + for (const origin of ["http://example.com", "https://user:secret@example.com", "https://example.com/path", "https://example.com?key=secret"]) { + assert.throws(() => toolDefinitions(origin)); + } + const tools = toolDefinitions("https://example.com"); + assert.equal(tools.length, 2); + assert.ok(tools.every(({ source }) => source.timeoutMs < 4000 && source.timeoutMs > WAIT_MS && !source.secret)); +}); + +test("rejects missing, wrong, stale, future and tampered signatures", async () => { + const { proof, request } = fixture(); + const valid = request("submit_request", { text: "synthetic request" }); + assert.equal((await proof.handle(valid.body, {})).status, 401); + assert.equal((await proof.handle(valid.body + " ", valid.headers)).status, 401); + for (const options of [{ signing: new Webhook(secret()) }, { timestamp: new Date(Date.now() - 360_000) }, { timestamp: new Date(Date.now() + 360_000) }]) { + assert.equal((await request("submit_request", { text: "test" }, options).send()).status, 401); + } + assert.equal(proof.evidence().acceptedRequests, 0); +}); + +test("deduplicates concurrent retries and rejects changed arguments and session substitution", async () => { + const { proof, request } = fixture(); + const original = request("submit_request", { text: "private phrase" }, { call: "same" }); + const responses = await Promise.all([original.send(), original.send(), original.send()]); + assert.ok(responses.every((response) => response.body.requestCount === 1)); + assert.equal((await request("submit_request", { text: "changed" }, { call: "same" }).send()).status, 409); + assert.equal((await request("submit_request", { text: "test" }, { session: "other" }).send()).status, 403); + assert.equal(proof.evidence().acceptedRequests, 1); + assert.ok(!JSON.stringify(proof.evidence()).includes("private phrase")); +}); + +test("waits 60 seconds, admits follow-ups to one job and distinguishes result retrieval from speech", async () => { + let clock = 0; + const waits = []; + const { proof, request } = fixture({ now: () => clock, wait: async (duration) => { waits.push(duration); clock += duration; } }); + assert.equal((await request("get_updates", { cursor: 0 }).send()).status, 409); + assert.equal((await request("submit_request", { text: "start" }).send()).body.status, "pending"); + assert.equal((await request("get_updates", { cursor: 1 }).send()).status, 400); + let response; + for (let i = 0; i < 24; i++) { + if (i === 6) assert.equal((await request("submit_request", { text: "followup" }).send()).body.requestCount, 2); + response = await request("get_updates", { cursor: 0 }).send(); + if (i < 23) assert.equal(response.body.status, "pending"); + } + assert.equal(clock, 60_000); + assert.ok(waits.every((duration) => duration <= 2500)); + assert.equal(response.body.status, "completed"); + assert.match(response.body.updates[0].text, /Verification number \d{4}\. Follow-up count 1/); + assert.deepEqual((await request("get_updates", { cursor: 1 }).send()).body.updates, []); + assert.equal(proof.evidence().liveQualification, "not_established"); + assert.equal(proof.evidence().resultReturned, true); +}); + +test("close during a bounded wait prevents further result delivery", async () => { + let release; + const { proof, request } = fixture({ wait: () => new Promise((resolve) => { release = resolve; }) }); + await request("submit_request", { text: "start" }).send(); + const polling = request("get_updates", { cursor: 0 }).send(); + await Promise.resolve(); + proof.close(); + release(); + assert.equal((await polling).status, 410); + assert.equal((await request("submit_request", { text: "late" }).send()).status, 410); +}); + +test("expires the single-run receiver and caps authenticated deliveries", async () => { + let clock = 0; + const { proof, request } = fixture({ now: () => clock }); + for (let i = 0; i < 500; i++) { + assert.equal((await request("submit_request", { text: "synthetic" }).send()).status, 200); + } + assert.equal((await request("submit_request", { text: "over limit" }).send()).status, 429); + clock = 15 * 60_000; + assert.equal((await request("submit_request", { text: "expired" }).send()).status, 410); + assert.equal(proof.evidence().acceptedRequests, 500); +}); + +test("rejects invalid args and bodies before accepting work", async () => { + const { proof, request } = fixture(); + for (const [tool, args] of [["unknown", {}], ["submit_request", { text: "" }], ["submit_request", { text: "ok", authority: "admin" }], ["get_updates", { cursor: -1 }], ["get_updates", { cursor: 0.5 }]]) { + assert.equal((await request(tool, args).send()).status, 400); + } + assert.equal((await proof.handle("x".repeat(BODY_LIMIT + 1), {})).status, 413); + assert.equal(proof.evidence().acceptedRequests, 0); +}); + +test("HTTP surface only accepts JSON on the signed tool route", async (t) => { + const { proof, request } = fixture(); + const server = createProofServer(proof); + t.after(() => { server.closeAllConnections(); server.close(); }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const base = `http://127.0.0.1:${server.address().port}`; + assert.equal((await fetch(`${base}/health`)).status, 200); + assert.equal((await fetch(`${base}/api/companies`)).status, 404); + assert.equal((await fetch(`${base}${TOOL_PATH}`, { method: "POST", body: "{}" })).status, 415); + const signed = request("submit_request", { text: "start" }); + const response = await fetch(`${base}${TOOL_PATH}`, { method: "POST", headers: { ...signed.headers, "content-type": "application/json" }, body: signed.body }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.equal((await response.json()).requestCount, 1); +}); diff --git a/tests/speko-voice-proof/summarize-run.mjs b/tests/speko-voice-proof/summarize-run.mjs new file mode 100644 index 00000000000..b5090b2eff3 --- /dev/null +++ b/tests/speko-voice-proof/summarize-run.mjs @@ -0,0 +1,49 @@ +/** Publish only an allowlisted, content-free view of a private synthetic run. */ +import { readFile, writeFile } from "node:fs/promises"; + +const [input, output] = process.argv.slice(2); +if (!input || !output) throw new Error("Usage: node summarize-run.mjs private/report.json new-report.json"); +const run = JSON.parse(await readFile(input, "utf8")); +const eventKinds = new Set([ + "session_created", "remote_audio_subscribed", "input_start", "input_end", + "request_accepted", "followup_accepted", "poll_returned", "result_ready", + "result_returned_to_tool", "tool_rejected", "transcript_received", + "script_completed", "script_failed", "media_disconnected", "session_end_requested", + "session_end_failed", "audio_stream_closed", "cleanup_failed", + "application_notification_sent", + "application_notifier_joined", "audio_activity_started", + "interruption_requested", +]); +const events = run.events.filter((event) => eventKinds.has(event.kind)).map((event) => { + const item = { kind: event.kind, elapsedMs: event.elapsedMs }; + for (const key of ["requestCount", "cursor", "jobElapsedMs", "characters", "verification", "toolMarkup"]) { + if (["number", "boolean"].includes(typeof event[key])) item[key] = event[key]; + } + if (["start", "followup"].includes(event.label)) item.label = event.label; + return item; +}); +const cleanupFailureCount = Math.max(run.cleanupFailures?.length ?? 0, run.events.filter(event => event.kind === "cleanup_failed").length); +const report = { + date: run.startedAt ?? "2026-09-11 (exact start available from provider session)", + sourceDigests: run.sourceDigests ? Object.fromEntries( + ["live-client.mjs", "proof.mjs", "media-failure.mjs"] + .filter((name) => /^[a-f0-9]{64}$/.test(run.sourceDigests[name] ?? "")) + .map((name) => [name, run.sourceDigests[name]]), + ) : null, + sessionId: run.sessionId, + input: run.mode, + applicationNotification: run.notifyReady === true, + applicationNotifierWebJoin: run.notifyViaJoin === true, + interruptAcknowledgment: run.interruptAcknowledgment === true, + llmConstraint: run.provider, + status: run.failure || cleanupFailureCount ? "failed" : "requires_audio_review", + cleanupFailureCount, + failure: run.failure ? "Scenario did not complete; see qualification report for diagnosis" : cleanupFailureCount ? "Run cleanup did not complete; see private evidence for diagnosis" : null, + acceptedRequests: run.proof.acceptedRequests, + resultReturned: run.proof.resultReturned, + synthesizedAudioReceived: run.audioFrames > 0, + // A transcript event and RTP frames cannot establish that the answer was heard. + confirmedResultPlayback: false, + events, +}; +await writeFile(output, `${JSON.stringify(report, null, 2)}\n`, { flag: "wx" }); diff --git a/tests/speko-voice-proof/summarize-run.test.mjs b/tests/speko-voice-proof/summarize-run.test.mjs new file mode 100644 index 00000000000..00a1232de8d --- /dev/null +++ b/tests/speko-voice-proof/summarize-run.test.mjs @@ -0,0 +1,54 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +test("shared reports exclude private content and never promote transport to playback proof", async () => { + const dir = await mkdtemp(join(tmpdir(), "speko-redaction-")); + try { + const input = join(dir, "private.json"); + const output = join(dir, "public.json"); + await writeFile(input, JSON.stringify({ + sessionId: "11111111-1111-4111-8111-111111111111", mode: "text", provider: "openai:gpt-4.1", + failure: null, audioFrames: 42, transportToken: "never-publish-token", + syntheticTranscript: [{ text: "never-publish-transcript" }], + sourceDigests: { "proof.mjs": "a".repeat(64), "media-failure.mjs": "b".repeat(64), accidentalKey: "never-publish-key" }, + proof: { acceptedRequests: 2, resultReturned: true, expectedSyntheticResult: "never-publish-phrase" }, + events: [ + { kind: "request_accepted", elapsedMs: 10, requestCount: 1, text: "never-publish-caller" }, + { kind: "script_failed", elapsedMs: 20, reason: "never-publish-error-detail" }, + { kind: "unknown_future_event", elapsedMs: 30, text: "never-publish-future" }, + ], + })); + const script = new URL("./summarize-run.mjs", import.meta.url); + const run = () => spawnSync(process.execPath, [script.pathname, input, output], { encoding: "utf8" }); + assert.equal(run().status, 0); + const text = await readFile(output, "utf8"); + assert.equal(text.includes("never-publish"), false); + const report = JSON.parse(text); + assert.equal(report.status, "requires_audio_review"); + assert.equal(report.confirmedResultPlayback, false); + assert.equal(report.resultReturned, true); + assert.equal(report.events.length, 2); + assert.deepEqual(report.sourceDigests, { "proof.mjs": "a".repeat(64), "media-failure.mjs": "b".repeat(64) }); + assert.notEqual(run().status, 0, "must refuse to overwrite evidence"); + assert.equal(await readFile(output, "utf8"), text); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shared reports mark cleanup failures without leaking private diagnostics", async () => { + const dir = await mkdtemp(join(tmpdir(), "speko-cleanup-redaction-")); + try { + const input = join(dir, "private.json"), output = join(dir, "public.json"); + await writeFile(input, JSON.stringify({failure: null, audioFrames: 1, proof: {acceptedRequests: 1, resultReturned: true}, cleanupFailures: [{operation: "source.close", reason: "secret detail"}], events: [{kind: "cleanup_failed", elapsedMs: 25, operation: "private operation"}]})); + assert.equal(spawnSync(process.execPath, [new URL("./summarize-run.mjs", import.meta.url).pathname, input, output]).status, 0); + const text = await readFile(output, "utf8"), report = JSON.parse(text); + assert.equal(report.status, "failed"); assert.equal(report.cleanupFailureCount, 1); + assert.equal(report.events[0].kind, "cleanup_failed"); + assert.equal(text.includes("secret detail"), false); assert.equal(text.includes("private operation"), false); + } finally {await rm(dir, {recursive: true, force: true});} +}); diff --git a/tests/storybook-visual/playwright.config.ts b/tests/storybook-visual/playwright.config.ts index e778bc7f16d..49ee969a17b 100644 --- a/tests/storybook-visual/playwright.config.ts +++ b/tests/storybook-visual/playwright.config.ts @@ -2,6 +2,7 @@ import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "@playwright/test"; +const storybookPort = Number(process.env.STORYBOOK_VISUAL_PORT ?? 6106); const testDir = dirname(fileURLToPath(import.meta.url)); const snapshotDir = process.env.STORYBOOK_VISUAL_SNAPSHOT_DIR ? isAbsolute(process.env.STORYBOOK_VISUAL_SNAPSHOT_DIR) @@ -47,11 +48,11 @@ export default defineConfig({ // JS-driven tickers/timers key off prefers-reduced-motion for // deterministic captures (CSS animations are already disabled). reducedMotion: "reduce", - baseURL: process.env.STORYBOOK_TEST_BASE_URL ?? "http://localhost:6106", + baseURL: process.env.STORYBOOK_TEST_BASE_URL ?? `http://localhost:${storybookPort}`, }, webServer: process.env.STORYBOOK_TEST_BASE_URL ? undefined : { - command: "node ../../scripts/serve-storybook-static.mjs --port 6106", - url: "http://localhost:6106/index.json", + command: `node ../../scripts/serve-storybook-static.mjs --port ${storybookPort}`, + url: `http://localhost:${storybookPort}/index.json`, reuseExistingServer: true, timeout: 30_000, }, diff --git a/tests/storybook-visual/speko-voice.spec.ts b/tests/storybook-visual/speko-voice.spec.ts new file mode 100644 index 00000000000..f935a97aab6 --- /dev/null +++ b/tests/storybook-visual/speko-voice.spec.ts @@ -0,0 +1,86 @@ +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { resolve } from "node:path"; +import { test, expect, type Page } from "@playwright/test"; + +const require = createRequire(import.meta.url); +const root = resolve(import.meta.dirname, "../.."); +const addon = require.resolve("@storybook/addon-a11y/package.json", { paths: [resolve(root, "ui")] }); +const axePath = require.resolve("axe-core/axe.min.js", { paths: [addon] }); +const index = JSON.parse(readFileSync(resolve(root, "ui/storybook-static/index.json"), "utf8")); +const stories = (Object.values(index.entries) as { id: string; type: string }[]).filter((entry) => entry.type === "story" && entry.id.startsWith("connections-speko-")); +async function protectStoryOrigin(page: Page, baseURL: string, external: string[]) { + const storyOrigin = new URL(baseURL).origin; + await page.route(/^https?:\/\//, (route) => { + const url = new URL(route.request().url()); + if (url.origin !== storyOrigin) { + external.push(url.hostname); + return route.abort(); + } + return route.fallback(); + }); +} + +test("Speko network isolation permits hosted Storybook and blocks providers", async ({ page }) => { + const baseURL = "https://storybook.example.invalid"; + const external: string[] = []; + const served: string[] = []; + await page.route(/^https?:\/\//, (route) => { + served.push(route.request().url()); + return route.fulfill({ contentType: "text/html", body: "Hosted fixture" }); + }); + await protectStoryOrigin(page, baseURL, external); + await page.goto(`${baseURL}/iframe.html`); + await expect(page).toHaveTitle("Hosted fixture"); + expect(external).toEqual([]); + expect(await page.evaluate(async (origin) => (await fetch(`${origin}/asset`)).ok, baseURL)).toBe(true); + expect(await page.evaluate(async () => { + try { await fetch("https://api.speko.dev/v1/calls"); return false; } + catch { return true; } + })).toBe(true); + expect(external).toEqual(["api.speko.dev"]); + expect(served).toEqual([`${baseURL}/iframe.html`, `${baseURL}/asset`]); +}); + +for (const story of stories) for (const theme of ["light", "dark"]) for (const width of [390, 1200]) { + test(`Speko accessibility ${story.id} ${theme} ${width}`, async ({ page }, info) => { + const errors: string[] = [], external: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await protectStoryOrigin(page, info.project.use.baseURL!, external); + await page.addInitScript(() => { + Object.defineProperty(navigator.mediaDevices, "getUserMedia", { value: () => { throw new Error("Story attempted real microphone access"); } }); + }); + await page.setViewportSize({ width, height: 844 }); + await page.goto(`/iframe.html?id=${story.id}&viewMode=story&globals=theme:${theme}`); + await expect(page.locator("#storybook-root")).not.toBeEmpty({ timeout: 15_000 }); + await expect(page.locator(".sb-errordisplay")).not.toBeVisible(); + // Storybook's play function exercises the actual controls and controller. + await page.waitForFunction((id) => document.body.dataset.spekoStoryReady === id, story.id); + expect(await page.locator("body").getAttribute("data-speko-story-error")).toBeNull(); + if (story.id === "connections-speko-call-history--keyboard-disclosure") { + const summary = page.locator("#storybook-root summary"); + await summary.press("Enter"); + await expect(page.getByRole("link", {name: "Conversation task", exact: true})).toBeVisible(); + await summary.press("Enter"); + await expect(page.getByRole("link", {name: "Conversation task", exact: true})).not.toBeVisible(); + } + await page.evaluate(() => document.fonts.ready); + await page.addScriptTag({ path: axePath }); + let violations: unknown[] = []; + // The addon also scans after play. Wait for that scan rather than racing + // its shared axe instance; retry only the explicit concurrent-run signal. + await expect.poll(async () => page.evaluate(async () => { + try { + const result = await (window as any).axe.run({ include: ["#storybook-root", "[role=dialog]"] }, { runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "wcag21aa"] } }); + return { complete: true, violations: result.violations.map((v: any) => ({ id: v.id, impact: v.impact, nodes: v.nodes.map((n: any) => n.target) })) }; + } catch (error) { + if (error instanceof Error && error.message.startsWith("Axe is already running")) return { complete: false, violations: [] }; + throw error; + } + }).then((result) => { violations = result.violations; return result.complete; })).toBe(true); + expect(violations).toEqual([]); + expect(errors).toEqual([]); expect(external).toEqual([]); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + await page.screenshot({ path: info.outputPath(`${story.id}-${theme}-${width}.png`), fullPage: true }); + }); +} diff --git a/tests/storybook-visual/storybook-visual.spec.ts b/tests/storybook-visual/storybook-visual.spec.ts index eb1ec09eca6..ff80d457e3b 100644 --- a/tests/storybook-visual/storybook-visual.spec.ts +++ b/tests/storybook-visual/storybook-visual.spec.ts @@ -51,11 +51,16 @@ async function renderStory(page: Page, storyId: string, theme: (typeof THEMES)[n await page.addInitScript(`{ const fixedNow = ${FIXED_TIME.getTime()}; const RealDate = Date; + // Speko play functions exercise quiet-period gates. Keep a fixed calendar + // origin while advancing elapsed time, so settling those gates is real. + const advanceDate = ${storyId.startsWith("connections-speko-")}; + const began = performance.now(); + const stableNow = () => fixedNow + (advanceDate ? Math.floor(performance.now() - began) : 0); class FixedDate extends RealDate { constructor(...args) { - if (args.length === 0) { super(fixedNow); } else { super(...args); } + if (args.length === 0) { super(stableNow()); } else { super(...args); } } - static now() { return fixedNow; } + static now() { return stableNow(); } } FixedDate.parse = RealDate.parse; FixedDate.UTC = RealDate.UTC; @@ -77,6 +82,10 @@ async function renderStory(page: Page, storyId: string, theme: (typeof THEMES)[n }); const errored = await page.locator(".sb-show-errordisplay").count(); expect(errored, `story ${storyId} threw during render`).toBe(0); + if (storyId.startsWith("connections-speko-")) { + await page.waitForFunction((id) => document.body.dataset.spekoStoryReady === id, storyId); + expect(await page.locator("body").getAttribute("data-speko-story-error")).toBeNull(); + } await page.evaluate(() => document.fonts.ready.then(() => undefined)); const settleMs = EXTRA_SETTLE_MS[storyId]; if (settleMs) await page.waitForTimeout(settleMs);