From 92dbf39e6ff057b292aaf498824c7f8001e61c93 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:44:45 -0600 Subject: [PATCH] feat(analytics): attribute agent work and onboarding steps to a person on Cloud (#22260) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Paperclip Cloud sends product events straight to PostHog: server events from `server/src/services/posthog-direct/` and browser events from `ui/src/lib/analytics/` > - A person-level funnel goes from sign-up to the first finished task > - Agent-driven events (`agent.task_run`, `agent.task_completed`, `agent.first_heartbeat`) used `workspace:` as the distinct id, so the funnel broke at those steps > - The in-workspace onboarding wizard sent no step events, so we could not see where people stop before their first agent > - This pull request attributes agent work to the responsible person on the Cloud direct path, and adds consent-gated onboarding step events in the browser > - The benefit is one PostHog person per customer across sign-up, onboarding, agent runs, and task completion. Self-hosted output does not change. ## Linked Issues or Issue Description **Subsystem affected** Cloud product analytics: Cloud direct PostHog delivery (server) and browser product analytics (UI onboarding wizard). **Problem or motivation** A PostHog funnel from sign-up to the first finished task cannot follow one person. Agent-driven events run in the background, with no signed-in request, so they use the workspace id as the distinct id. The onboarding wizard also sends no events, so we cannot see which step people leave. **Proposed solution** Give background emit sites a narrow scope that names the person responsible for the work. Use the task creator first, then the task's responsible user, then the run owner. Keep the signed-in request user first. Add `attributed_via` to every Cloud direct event. Add four browser events to the onboarding wizard. Send them through the existing analytics module, so the consent rules stay the same. **Alternatives considered** A lookup in the transport at send time. We did not use it: it adds database reads to the telemetry path and loses the emit site's context. A new contract event for onboarding steps. We did not use it: the server does not see the wizard steps, and the UI must never send `PAPERCLIP_EVENTS` names. **Additional context** Self-hosted installs keep the collector path. The collector envelope stays byte-identical (a test checks it). ## What Changed - `server/src/telemetry-company-scope.ts`: add `withTelemetryActor({ companyId, person }, fn)`, `scopedTelemetryPerson()`, and `resolveTelemetryPerson()`. The resolver picks the task creator, then the task's responsible user, then the run owner. It ignores `local-board`. `withTelemetryCompany` now calls `withTelemetryActor`. - `server/src/services/posthog-direct/request-context.ts`: a signed-in request user wins (`attributed_via: request`). Without one, the scoped person is used. - `server/src/services/posthog-direct/event-shape.ts`: every Cloud direct event (including `cloud.workspace_started`) has `attributed_via` (`request`, `task_creator`, `task_assigner`, `run_owner`, or `none`). An event with a person keeps today's person-event shape (no `$process_person_profile: false`). - `server/src/services/agent-task-run-telemetry.ts`: `agent.task_run` reads the task creator and responsible user in the existing agent query (one left join, no new query), and the run's `responsibleUserId`. - `server/src/routes/issues.ts`: `agent.task_completed` uses the issue row that the route already has, and the agent actor's `onBehalfOfUserId` as the run owner. - `server/src/services/heartbeat/agent-status.ts`: `agent.first_heartbeat` uses the agent's `createdByUserId` (already loaded) as the run owner. - `ui/src/lib/analytics/product-analytics.ts`: add `captureProductAnalyticsEvent()`. It captures only with a browser analytics config and granted consent. Events from before posthog-js loads wait in a queue (at most 50) and keep their time. Withdrawn consent or a change of signed-in person drops the queue. - `ui/src/lib/analytics/onboarding-analytics.ts` and `ui/src/components/OnboardingWizard.tsx`: capture `onboarding_opened`, `onboarding_step_viewed`, `onboarding_step_completed` (with `step` and, for `connect_model` and `launch`, `outcome`), and `onboarding_dismissed`. `step` is `create_agent`, `connect_model`, `review`, or `launch`. No event has free text. - Docs: `packages/shared/src/telemetry/README.md` (Cloud Direct Delivery) and `doc/cloud-browser-analytics.md` (new "Onboarding events" section). ## Verification - `server/src/__tests__/posthog-direct-telemetry.test.ts`: attribution matrix (creator, assigner, run owner, none stays on `workspace:` with `$process_person_profile: false`), the request user wins over the scope, and the self-hosted collector envelope is byte-identical with a person scope. - `server/src/__tests__/agent-task-run-telemetry.test.ts` (embedded Postgres): `agent.task_run` scope for creator, assigner, run owner, and nobody; another company's task is never read. - `server/src/__tests__/issue-telemetry-routes.test.ts`: `agent.task_completed` scope for creator and assigner. - `server/src/__tests__/heartbeat-process-recovery.test.ts`: `agent.first_heartbeat` scope is the agent owner. - `ui/src/components/OnboardingWizard.test.tsx`: a launch for the old company that settles after a switch to another company does not end the new analytics session. - `ui/src/lib/analytics/onboarding-analytics.test.tsx`: each transition fires once, held events drop on a person change, a stale opening step is not a view, dismissal fires once, nothing is captured when analytics is off or consent is `denied` or `ask`, and queued events keep their time. - `ui/src/lib/analytics/product-analytics-events.test.ts` still passes (the UI never captures a `PAPERCLIP_EVENTS` name). - Local Cloud-mode run: an isolated instance (own Postgres, trusted Cloud tenant headers, `PAPERCLIP_POSTHOG_HOST` set to a local capture server, never real PostHog). A stub stood in for posthog-js and sent through the `/_ph` proxy. - A user created a task and assigned it to the agent. `agent.task_run` arrived with `distinct_id: paperclip-id:local-user-1`, `attributed_via: task_creator`, the raw `task_id`, and no `$process_person_profile`. (The run itself failed locally because the Cloud runner is not available. The event still shows the attribution.) - `agent.first_heartbeat` arrived with the same person and `attributed_via: run_owner`. `server.started` kept `workspace:stack-local` with `attributed_via: none`. - Headless Chromium walked the wizard: `onboarding_opened` (create_agent), `onboarding_step_viewed` (create_agent), `onboarding_step_completed` (create_agent), `onboarding_step_viewed` (connect_model), `onboarding_step_completed` (connect_model, `api_key`), `onboarding_step_viewed` (review), `onboarding_step_completed` (review), `onboarding_step_completed` (launch, `succeeded`). Each fired once, with the signed-in user as the distinct id. Events from before posthog-js loaded arrived with their original time. - A second walk left the onboarding page on the connect step and sent one `onboarding_dismissed` (connect_model). ## Risks - Low risk for self-hosted: the collector path never reads the new scope, and a test checks the envelope bytes. - Cloud: agent events move from the workspace person to a real person. Insights that count `workspace:` distinct ids for these events will change. `attributed_via` lets a query filter or split them. - `task_assigner` uses the task's responsible user. Paperclip does not store who assigned a task, so this is the person who owns or delegated the task. - Privacy: the new data is Cloud-only and uses ids that Cloud events already carry. No new free text, email, or name goes on an event. **Privacy review requested** for the Cloud attribution change (per `rules/data-paths.mdc`). The generated contract does not change: `attributed_via` is a Cloud-only property, like `stack_id` and `user_id`. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits, headless browser). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/cloud-browser-analytics.md | 20 ++ packages/shared/src/telemetry/README.md | 13 + .../agent-task-run-telemetry.test.ts | 44 ++- .../heartbeat-process-recovery.test.ts | 10 +- .../__tests__/issue-telemetry-routes.test.ts | 34 +++ .../posthog-direct-event-shape.test.ts | 2 + .../posthog-direct-telemetry.test.ts | 123 ++++++++- server/src/routes/issues.ts | 9 +- .../src/services/agent-task-run-telemetry.ts | 33 ++- server/src/services/heartbeat/agent-status.ts | 5 +- .../services/posthog-direct/event-shape.ts | 2 + .../posthog-direct/request-context.ts | 17 +- server/src/telemetry-company-scope.ts | 52 +++- .../runner-e2e/source-sdk-public-inputs.json | 2 +- ui/src/components/OnboardingWizard.test.tsx | 25 ++ ui/src/components/OnboardingWizard.tsx | 14 + .../analytics/onboarding-analytics.test.tsx | 260 ++++++++++++++++++ ui/src/lib/analytics/onboarding-analytics.ts | 92 +++++++ ui/src/lib/analytics/product-analytics.ts | 31 +++ 19 files changed, 766 insertions(+), 22 deletions(-) create mode 100644 ui/src/lib/analytics/onboarding-analytics.test.tsx create mode 100644 ui/src/lib/analytics/onboarding-analytics.ts diff --git a/doc/cloud-browser-analytics.md b/doc/cloud-browser-analytics.md index 7d34cf3e94b..4bd172a0dc5 100644 --- a/doc/cloud-browser-analytics.md +++ b/doc/cloud-browser-analytics.md @@ -233,6 +233,26 @@ This value is used only for location, never for limits. - API requests to our own host carry PostHog tracing headers, so server events link to the browser session. +## Onboarding events + +The in-workspace onboarding wizard (`ui/src/components/OnboardingWizard.tsx`) +captures four browser events through `ui/src/lib/analytics/onboarding-analytics.ts`. +They follow the same consent rules as every other browser event: without a +browser analytics config or with consent not granted, nothing is captured. +Events from before posthog-js loads wait in a short queue (at most 50) and keep +their original time. Withdrawing consent or a change of signed-in person drops +the queue. + +| Event | When | Properties | +| --- | --- | --- | +| `onboarding_opened` | The wizard opens, once per open. Auto-open (`useAgentlessOnboardingAutoOpen`), a button, and the `/onboarding` route all count. | `step` when the wizard opens on an agent step | +| `onboarding_step_viewed` | A step is shown, once per step change. | `step` | +| `onboarding_step_completed` | The customer moves forward one step, presses Launch on the review step, or Launch finishes. | `step`; `outcome` for `connect_model` (`subscription` or `api_key`) and for `launch` (`succeeded` or `failed`) | +| `onboarding_dismissed` | The wizard closes before Launch finishes, for example when the customer leaves the onboarding page. | `step` when closed on an agent step | + +`step` is one of `create_agent`, `connect_model`, `review`, or `launch`. The +organization-naming steps have no `step` value. No event carries free text. + ## Privacy Session replay shows text by default, the way PostHog normally works. Text is diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md index eaa75efee24..99c1d1eddf1 100644 --- a/packages/shared/src/telemetry/README.md +++ b/packages/shared/src/telemetry/README.md @@ -150,6 +150,19 @@ exposes the `TelemetryTransport` seam on `TelemetryClient`: reports on. Only the direct transport reads this scope, and it uses it only when no live request supplies a company. The collector envelope and the dimensions do not change. +- Agent-driven events (`agent.task_run`, `agent.task_completed`, and + `agent.first_heartbeat`) also name the person responsible for the work, with + `withTelemetryActor({ companyId, person }, fn)` and + `resolveTelemetryPerson()` from the same file. The person is the user who + created the task, else the task's responsible user (who assigned or + delegated it), else the user who owns the run (for `agent.first_heartbeat`, + the user who created the agent). The emit site reads these ids from rows it + already loads; the transport does no database reads. A signed-in user on a + live request always wins over this scope. With no person, the event stays on + `workspace:` with `$process_person_profile: false`. Every direct + event carries `attributed_via`: `request`, `task_creator`, `task_assigner`, + `run_owner`, or `none`. The collector path never reads this scope, so the + self-hosted envelope does not change. Cloud is Paperclip's own service under the Cloud terms, so its events carry real identifiers: raw agent and task ids, the stack id, and the company and diff --git a/server/src/__tests__/agent-task-run-telemetry.test.ts b/server/src/__tests__/agent-task-run-telemetry.test.ts index abf11fd70fb..46e043463bf 100644 --- a/server/src/__tests__/agent-task-run-telemetry.test.ts +++ b/server/src/__tests__/agent-task-run-telemetry.test.ts @@ -1,12 +1,12 @@ import { randomBytes, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { agents, companies, createDb, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { eq } from "drizzle-orm"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -import { scopedTelemetryCompanyId } from "../telemetry-company-scope.js"; +import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../telemetry-company-scope.js"; const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); const mockTrackAgentTaskRun = vi.hoisted(() => vi.fn()); @@ -44,6 +44,7 @@ describeEmbeddedPostgres("emitAgentTaskRun", () => { afterEach(async () => { vi.clearAllMocks(); await db.delete(heartbeatRuns); + await db.delete(issues); await db.delete(agents); await db.delete(companies); }); @@ -120,6 +121,45 @@ describeEmbeddedPostgres("emitAgentTaskRun", () => { expect(scopedTelemetryCompanyId()).toBeUndefined(); }); + it.each([ + { name: "task creator", issue: { createdByUserId: "user-creator", responsibleUserId: "user-assigner" }, runOwner: "user-owner", expected: { userId: "user-creator", attributedVia: "task_creator" } }, + { name: "task assigner", issue: { createdByUserId: null, responsibleUserId: "user-assigner" }, runOwner: "user-owner", expected: { userId: "user-assigner", attributedVia: "task_assigner" } }, + { name: "run owner", issue: { createdByUserId: null, responsibleUserId: null }, runOwner: "user-owner", expected: { userId: "user-owner", attributedVia: "run_owner" } }, + { name: "nobody", issue: { createdByUserId: null, responsibleUserId: null }, runOwner: null, expected: undefined }, + ])("tracks agent.task_run for the person responsible: $name", async ({ issue, runOwner, expected }) => { + await seedCompanyAndAgent(); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Write the report", assigneeAgentId: agentId, createdByAgentId: issue.createdByUserId ? null : agentId, ...issue }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + responsibleUserId: runOwner, contextSnapshot: { issueId } }).returning(); + const people: unknown[] = []; + const recordPerson = () => people.push(scopedTelemetryPerson()); + mockTrackAgentTaskRun.mockImplementationOnce(recordPerson).mockImplementationOnce(recordPerson); + + await emitAgentTaskRun(db, run); + await emitAgentTaskRunById(db, { runId: run.id, companyId }); + + expect(people).toEqual([expected, expected]); + expect(mockTrackAgentTaskRun).toHaveBeenCalledWith(mockTelemetryClient, expect.objectContaining({ taskId: issueId })); + }); + + it("does not read another company's task for attribution", async () => { + await seedCompanyAndAgent(); + const otherCompanyId = randomUUID(); + await db.insert(companies).values({ id: otherCompanyId, name: "Other", issuePrefix: `O${otherCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false }); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId: otherCompanyId, title: "Other task", createdByUserId: "user-other" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", contextSnapshot: { issueId } }).returning(); + const people: unknown[] = []; + mockTrackAgentTaskRun.mockImplementationOnce(() => people.push(scopedTelemetryPerson())); + + await emitAgentTaskRun(db, run); + + expect(people).toEqual([undefined]); + await db.delete(issues); + await db.delete(companies).where(eq(companies.id, otherCompanyId)); + }); + it("emits the same event from a small scoped run read without loading execution payloads", async () => { await seedCompanyAndAgent(); const payload = randomBytes(128 * 1024).toString("base64"); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index a0865338dcf..8422a92ce12 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -124,7 +124,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -import { scopedTelemetryCompanyId } from "../telemetry-company-scope.js"; +import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../telemetry-company-scope.js"; import type { EnvironmentLease } from "@paperclipai/shared"; import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js"; import { runningProcesses, type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts"; @@ -10040,9 +10040,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { agentStatus: "running", includeIssue: false, }); + await db.update(agents).set({ createdByUserId: "paperclip-id:agent-owner" }).where(eq(agents.id, agentId)); const heartbeat = heartbeatService(db); const scopedCompanies: Array = []; - mockTrackAgentFirstHeartbeat.mockImplementationOnce(() => scopedCompanies.push(scopedTelemetryCompanyId())); + const scopedPeople: unknown[] = []; + mockTrackAgentFirstHeartbeat.mockImplementationOnce(() => { + scopedCompanies.push(scopedTelemetryCompanyId()); + scopedPeople.push(scopedTelemetryPerson()); + }); await heartbeat.cancelRun(runId); @@ -10054,6 +10059,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }), ); expect(scopedCompanies).toEqual([companyId]); + expect(scopedPeople).toEqual([{ userId: "paperclip-id:agent-owner", attributedVia: "run_owner" }]); }); it("preserves first-heartbeat telemetry after a timer interval claim", async () => { diff --git a/server/src/__tests__/issue-telemetry-routes.test.ts b/server/src/__tests__/issue-telemetry-routes.test.ts index bf29f4f5059..04e75132724 100644 --- a/server/src/__tests__/issue-telemetry-routes.test.ts +++ b/server/src/__tests__/issue-telemetry-routes.test.ts @@ -219,6 +219,40 @@ describe("issue telemetry routes", () => { expect(scopedCompanies).toEqual(["company-1"]); }, 10_000); + it.each([ + { name: "task creator", issue: { createdByUserId: "paperclip-id:creator", responsibleUserId: "paperclip-id:assigner" }, expected: { userId: "paperclip-id:creator", attributedVia: "task_creator" } }, + { name: "task assigner", issue: { createdByUserId: null, responsibleUserId: "paperclip-id:assigner" }, expected: { userId: "paperclip-id:assigner", attributedVia: "task_assigner" } }, + ])("attributes agent task-completed telemetry to the $name", async ({ issue, expected }) => { + const { scopedTelemetryPerson } = await import("../telemetry-company-scope.js"); + const people: unknown[] = []; + mockTrackAgentTaskCompleted.mockImplementationOnce(() => people.push(scopedTelemetryPerson())); + mockIssueService.update.mockImplementation(async (_id: string, patch: Record) => ({ + ...makeIssue("todo"), + ...issue, + ...patch, + })); + mockAgentService.getById.mockResolvedValue({ + id: "agent-1", + companyId: "company-1", + role: "engineer", + adapterType: "codex_local", + adapterConfig: {}, + }); + + const app = await createApp({ + type: "agent", + agentId: "agent-1", + companyId: "company-1", + runId: "22222222-2222-4222-8222-222222222222", + }); + const res = await request(app) + .patch("/api/issues/11111111-1111-4111-8111-111111111111") + .send({ status: "done" }); + + expect(res.status).toBe(200); + await vi.waitFor(() => expect(people).toEqual([expected])); + }, 10_000); + it("does not emit agent task-completed telemetry for board-driven completions", async () => { const app = await createApp({ type: "board", diff --git a/server/src/__tests__/posthog-direct-event-shape.test.ts b/server/src/__tests__/posthog-direct-event-shape.test.ts index 5ae8e2bf623..27addcdad86 100644 --- a/server/src/__tests__/posthog-direct-event-shape.test.ts +++ b/server/src/__tests__/posthog-direct-event-shape.test.ts @@ -59,6 +59,7 @@ describe("Cloud direct PostHog event shape", () => { const intended = { distinct_id: "workspace:stack-123", stack_id: "stack-123", + attributed_via: "none", environment: "staging", $groups: { workspace: "stack-123" }, ...("agent_id" in inputDims ? { agent_id: inputDims.agent_id } : {}), @@ -85,6 +86,7 @@ describe("Cloud direct PostHog event shape", () => { distinct_id: "user-1", user_id: "user-1", company_id: "company-1", + attributed_via: "request", $session_id: "sess-1", $window_id: "win-1", $groups: { workspace: "stack-123" }, diff --git a/server/src/__tests__/posthog-direct-telemetry.test.ts b/server/src/__tests__/posthog-direct-telemetry.test.ts index b35b0b1b3f6..cf4342c80bd 100644 --- a/server/src/__tests__/posthog-direct-telemetry.test.ts +++ b/server/src/__tests__/posthog-direct-telemetry.test.ts @@ -31,7 +31,7 @@ import { capturePostHogEventContext, postHogRequestContextMiddlewareForEnv, } from "../services/posthog-direct/request-context.js"; -import { withTelemetryCompany } from "../telemetry-company-scope.js"; +import { resolveTelemetryPerson, withTelemetryActor, withTelemetryCompany } from "../telemetry-company-scope.js"; import { serverVersion } from "../version.js"; const COLLECTOR = "http://collector.test/ingest"; @@ -159,6 +159,40 @@ describe("server telemetry sender selection", () => { expect(body).not.toContain("company"); }); + it("self-hosted: a background person scope leaves the collector envelope byte-identical", async () => { + const client = initTelemetry({ enabled: true }, { env: {} })!; + const person = resolveTelemetryPerson({ taskCreatedByUserId: "user-creator" }); + await withTelemetryActor({ companyId: "company-bg", person }, async () => { + await Promise.resolve(); + trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" }); + }); + await shutdownTelemetry(); + + expect(hoisted.posthogNodeLoads).toBe(0); + expect(fetchMock).toHaveBeenCalledTimes(1); + const body = String((fetchMock.mock.calls[0]![1] as RequestInit).body); + const parsed = JSON.parse(body); + expect(body).toBe( + JSON.stringify({ + app: "paperclip", + schemaVersion: "1", + installId: readState().installId, + version: serverVersion, + events: [ + { name: "server.started", occurredAt: parsed.events[0].occurredAt, dimensions: { os_platform: process.platform } }, + { + name: "agent.task_run", + occurredAt: parsed.events[1].occurredAt, + dimensions: { agent_id: "agent-1", state: "succeeded" }, + }, + ], + batchId: parsed.batchId, + }), + ); + expect(body).not.toContain("user-creator"); + expect(body).not.toContain("attributed_via"); + }); + it("self-hosted: ignores posthog-js tracing headers", async () => { expect(postHogRequestContextMiddlewareForEnv({})).toBeNull(); expect(postHogRequestContextMiddlewareForEnv({ PAPERCLIP_POSTHOG_PROJECT_TOKEN: "phc_test" })).toBeNull(); @@ -222,12 +256,14 @@ describe("server telemetry sender selection", () => { version: serverVersion, stack_id: "stack-123", install_hash: installHash, + attributed_via: "none", environment: "staging", distinct_id: "workspace:stack-123", $process_person_profile: false, }); for (const message of posthog.captured) { expect(message.properties?.environment, message.event).toBe("staging"); + if (message.event !== "$groupidentify") expect(message.properties?.attributed_via, message.event).toBe("none"); if (message.event !== "cloud.workspace_started") expect(message.properties).not.toHaveProperty("install_hash"); } expect(posthog.groups).toEqual([ @@ -432,4 +468,89 @@ describe("server telemetry sender selection", () => { expect(routines[0]!.properties).toMatchObject({ company_id: "company-9", user_id: "user-1" }); expect(routines[1]!.properties).toMatchObject({ company_id: "company-bg", user_id: "user-1" }); }); + + it("Cloud: resolves the person responsible for agent work in priority order", () => { + expect(resolveTelemetryPerson({ + taskCreatedByUserId: "user-creator", + taskAssignedByUserId: "user-assigner", + runOwnerUserId: "user-owner", + })).toEqual({ userId: "user-creator", attributedVia: "task_creator" }); + expect(resolveTelemetryPerson({ taskAssignedByUserId: "user-assigner", runOwnerUserId: "user-owner" })) + .toEqual({ userId: "user-assigner", attributedVia: "task_assigner" }); + expect(resolveTelemetryPerson({ taskCreatedByUserId: " ", runOwnerUserId: "user-owner" })) + .toEqual({ userId: "user-owner", attributedVia: "run_owner" }); + expect(resolveTelemetryPerson({ taskCreatedByUserId: "local-board", runOwnerUserId: null })).toBeNull(); + expect(resolveTelemetryPerson({})).toBeNull(); + }); + + it.each([ + { name: "task creator", input: { taskCreatedByUserId: "paperclip-id:creator", taskAssignedByUserId: "paperclip-id:assigner" }, distinctId: "paperclip-id:creator", via: "task_creator" }, + { name: "task assigner", input: { taskAssignedByUserId: "paperclip-id:assigner", runOwnerUserId: "paperclip-id:owner" }, distinctId: "paperclip-id:assigner", via: "task_assigner" }, + { name: "run owner", input: { runOwnerUserId: "paperclip-id:owner" }, distinctId: "paperclip-id:owner", via: "run_owner" }, + ])("Cloud: background agent work is attributed to the $name", async ({ input, distinctId, via }) => { + const posthog = fakePostHog(); + const client = initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory })!; + await withTelemetryActor({ companyId: "company-bg", person: resolveTelemetryPerson(input) }, async () => { + await new Promise((resolve) => setTimeout(resolve, 1)); + trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" }); + }); + await shutdownTelemetry(); + + const run = posthog.captured.find((message) => message.event === "agent.task_run")!; + expect(run.distinctId).toBe(distinctId); + expect(run.groups).toEqual({ workspace: "stack-123" }); + expect(run.properties).toMatchObject({ + company_id: "company-bg", + user_id: distinctId, + attributed_via: via, + distinct_id: distinctId, + }); + expect(run.properties).not.toHaveProperty("$process_person_profile"); + }); + + it("Cloud: background agent work with no person stays on the workspace", async () => { + const posthog = fakePostHog(); + const client = initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory })!; + withTelemetryActor({ companyId: "company-bg", person: resolveTelemetryPerson({}) }, () => + trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" })); + await shutdownTelemetry(); + + const run = posthog.captured.find((message) => message.event === "agent.task_run")!; + expect(run.distinctId).toBe("workspace:stack-123"); + expect(run.properties).toMatchObject({ attributed_via: "none", $process_person_profile: false }); + expect(run.properties).not.toHaveProperty("user_id"); + }); + + it("Cloud: the signed-in request user wins over a background person scope", async () => { + const posthog = fakePostHog(); + initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory }); + const app = express(); + app.use(postHogRequestContextMiddlewareForEnv(CLOUD_ENV)!); + app.use((req, _res, next) => { + if (req.path === "/agent") req.actor = { type: "agent", agentId: "agent-1", companyId: "company-1", source: "agent_jwt" }; + else req.actor = { type: "board", userId: "paperclip-id:request-user", source: "session" }; + next(); + }); + const person = resolveTelemetryPerson({ taskCreatedByUserId: "paperclip-id:creator" }); + app.post("/board", (_req, res) => { + withTelemetryActor({ companyId: "company-1", person }, () => + trackAgentTaskRun(getTelemetryClient()!, { agentId: "agent-1", state: "cancelled" })); + res.json({}); + }); + app.post("/agent", (_req, res) => { + withTelemetryActor({ companyId: "company-1", person }, () => + trackAgentTaskRun(getTelemetryClient()!, { agentId: "agent-1", state: "succeeded" })); + res.json({}); + }); + await request(app).post("/board").expect(200); + await request(app).post("/agent").expect(200); + await shutdownTelemetry(); + + const runs = posthog.captured.filter((message) => message.event === "agent.task_run"); + const byState = Object.fromEntries(runs.map((message) => [message.properties?.state, message])); + expect(byState.cancelled!.distinctId).toBe("paperclip-id:request-user"); + expect(byState.cancelled!.properties).toMatchObject({ attributed_via: "request" }); + expect(byState.succeeded!.distinctId).toBe("paperclip-id:creator"); + expect(byState.succeeded!.properties).toMatchObject({ attributed_via: "task_creator", company_id: "company-1" }); + }); }); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 6a115e1abf5..d6ddcf17cb5 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -208,7 +208,7 @@ import { } from "@paperclipai/shared"; import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry"; import { getTelemetryClient } from "../telemetry.js"; -import { withTelemetryCompany } from "../telemetry-company-scope.js"; +import { resolveTelemetryPerson, withTelemetryActor } from "../telemetry-company-scope.js"; import { isUniqueViolation } from "../db-errors.js"; import type { StorageService } from "../storage/types.js"; import { validate, validateIssueMutationBody } from "../middleware/validate.js"; @@ -16705,7 +16705,12 @@ export function issueRoutes( typeof actorAgent.adapterConfig?.model === "string" ? actorAgent.adapterConfig.model : undefined; - withTelemetryCompany(issue.companyId, () => trackAgentTaskCompleted(tc, { + const person = resolveTelemetryPerson({ + taskCreatedByUserId: issue.createdByUserId, + taskAssignedByUserId: issue.responsibleUserId, + runOwnerUserId: req.actor.type === "agent" ? req.actor.onBehalfOfUserId : null, + }); + withTelemetryActor({ companyId: issue.companyId, person }, () => trackAgentTaskCompleted(tc, { agentRole: actorAgent.role, agentId: actorAgent.id, adapterType: actorAgent.adapterType, diff --git a/server/src/services/agent-task-run-telemetry.ts b/server/src/services/agent-task-run-telemetry.ts index a80243d9363..16550af05d6 100644 --- a/server/src/services/agent-task-run-telemetry.ts +++ b/server/src/services/agent-task-run-telemetry.ts @@ -1,14 +1,17 @@ import { and, eq, sql } from "drizzle-orm"; -import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { agents, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { trackAgentTaskRun } from "@paperclipai/shared/telemetry"; import { parseObject } from "../adapters/utils.js"; import { logger } from "../middleware/logger.js"; import { getTelemetryClient } from "../telemetry.js"; -import { withTelemetryCompany } from "../telemetry-company-scope.js"; +import { resolveTelemetryPerson, withTelemetryActor } from "../telemetry-company-scope.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type TelemetryRun = Pick; + "id" | "companyId" | "agentId" | "status" | "startedAt" | "finishedAt" | "usageJson" | "contextSnapshot"> + & Partial>; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 @@ -60,13 +63,26 @@ export async function emitAgentTaskRun(db: Db, run: TelemetryRun): Promise const client = getTelemetryClient(); if (!client) return; + const taskId = + readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? + undefined; + const taskRowId = taskId && UUID_RE.test(taskId) ? taskId : null; + const agent = await db .select({ role: agents.role, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig, + taskCreatedByUserId: issues.createdByUserId, + taskResponsibleUserId: issues.responsibleUserId, }) .from(agents) + .leftJoin( + issues, + taskRowId + ? and(eq(issues.id, taskRowId), eq(issues.companyId, run.companyId)) + : sql`false`, + ) .where(eq(agents.id, run.agentId)) .then((rows) => rows[0] ?? null); @@ -76,11 +92,13 @@ export async function emitAgentTaskRun(db: Db, run: TelemetryRun): Promise readNonEmptyString(parseObject(agent?.adapterConfig).model) ?? undefined; - const taskId = - readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? - undefined; + const person = resolveTelemetryPerson({ + taskCreatedByUserId: agent?.taskCreatedByUserId, + taskAssignedByUserId: agent?.taskResponsibleUserId, + runOwnerUserId: run.responsibleUserId, + }); - withTelemetryCompany(run.companyId, () => trackAgentTaskRun(client, { + withTelemetryActor({ companyId: run.companyId, person }, () => trackAgentTaskRun(client, { agentId: run.agentId, state: run.status, ...(agent @@ -141,6 +159,7 @@ export async function emitAgentTaskRunById( startedAt: heartbeatRuns.startedAt, finishedAt: heartbeatRuns.finishedAt, usageJson: heartbeatRuns.usageJson, + responsibleUserId: heartbeatRuns.responsibleUserId, contextSnapshot: sql>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId')`, }) .from(heartbeatRuns) diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index 251b5794a6c..d08310b83e8 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -5,7 +5,7 @@ import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry"; import { getTelemetryClient } from "../../telemetry.js"; -import { withTelemetryCompany } from "../../telemetry-company-scope.js"; +import { resolveTelemetryPerson, withTelemetryActor } from "../../telemetry-company-scope.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { publishLiveEvent } from "../live-events.js"; import { finishedIdleRun, finishedIdleWake, finishedFailedIdleLease } from "../idle-work-proof.js"; @@ -117,7 +117,8 @@ export function createHeartbeatAgentStatus(db: Db) { if (!result?.updated) return; if (result.isFirstHeartbeat) { const client = getTelemetryClient(); - if (client) withTelemetryCompany(result.updated.companyId, () => trackAgentFirstHeartbeat(client, { agentRole: result.updated.role, agentId: result.updated.id })); + const person = resolveTelemetryPerson({ runOwnerUserId: result.updated.createdByUserId }); + if (client) withTelemetryActor({ companyId: result.updated.companyId, person }, () => trackAgentFirstHeartbeat(client, { agentRole: result.updated.role, agentId: result.updated.id })); } publishStatus(result.updated, outcome); } diff --git a/server/src/services/posthog-direct/event-shape.ts b/server/src/services/posthog-direct/event-shape.ts index 7b763028cb6..7616e6e1830 100644 --- a/server/src/services/posthog-direct/event-shape.ts +++ b/server/src/services/posthog-direct/event-shape.ts @@ -179,6 +179,7 @@ export function buildPostHogDirectEvent( stack_id: config.stackId, ...(context?.companyId ? { company_id: context.companyId } : {}), ...(context?.userId ? { user_id: context.userId } : {}), + attributed_via: context?.userId ? (context.attributedVia ?? "request") : "none", ...(config.environment ? { environment: config.environment } : {}), ...(context?.sessionId ? { $session_id: context.sessionId } : {}), ...(context?.windowId ? { $window_id: context.windowId } : {}), @@ -224,6 +225,7 @@ export function buildWorkspaceStartedEvent( ...(meta.version ? { version: meta.version } : {}), stack_id: config.stackId, install_hash: input.installHash, + attributed_via: "none", ...(config.environment ? { environment: config.environment } : {}), $groups: { [POSTHOG_WORKSPACE_GROUP_TYPE]: config.stackId }, distinct_id: distinctId, diff --git a/server/src/services/posthog-direct/request-context.ts b/server/src/services/posthog-direct/request-context.ts index a037e260594..9b376adba54 100644 --- a/server/src/services/posthog-direct/request-context.ts +++ b/server/src/services/posthog-direct/request-context.ts @@ -1,7 +1,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import type { Request, RequestHandler } from "express"; import { isPostHogDirectDeliveryEnabled, resolveCloudPostHogConfig } from "../cloud-posthog-config.js"; -import { scopedTelemetryCompanyId } from "../../telemetry-company-scope.js"; +import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../../telemetry-company-scope.js"; /** * Request-scoped analytics context for Paperclip Cloud direct delivery. @@ -21,8 +21,11 @@ type PostHogRequestStore = { active: boolean; }; +export type PostHogAttribution = "request" | "task_creator" | "task_assigner" | "run_owner" | "none"; + export type PostHogEventContext = { userId?: string; + attributedVia?: PostHogAttribution; companyId?: string; sessionId?: string; windowId?: string; @@ -103,11 +106,21 @@ export function capturePostHogEventContext(): PostHogEventContext | undefined { if (store?.active) { const userId = actingUserId(store.req); const companyId = knownCompanyId(store.req); - if (userId) context.userId = userId; + if (userId) { + context.userId = userId; + context.attributedVia = "request"; + } if (companyId) context.companyId = companyId; if (store.sessionId) context.sessionId = store.sessionId; if (store.windowId) context.windowId = store.windowId; } + if (!context.userId) { + const person = scopedTelemetryPerson(); + if (person) { + context.userId = person.userId; + context.attributedVia = person.attributedVia; + } + } if (!context.companyId) { const scopedCompanyId = scopedTelemetryCompanyId(); if (scopedCompanyId) context.companyId = scopedCompanyId; diff --git a/server/src/telemetry-company-scope.ts b/server/src/telemetry-company-scope.ts index 6922fcce34a..98a0f2c71f1 100644 --- a/server/src/telemetry-company-scope.ts +++ b/server/src/telemetry-company-scope.ts @@ -1,11 +1,57 @@ import { AsyncLocalStorage } from "node:async_hooks"; -const storage = new AsyncLocalStorage(); +export type TelemetryPersonAttribution = "task_creator" | "task_assigner" | "run_owner"; + +export type TelemetryPerson = { + userId: string; + attributedVia: TelemetryPersonAttribution; +}; + +type TelemetryScope = { + companyId?: string; + person?: TelemetryPerson; +}; + +const storage = new AsyncLocalStorage(); +const NON_PERSON_USER_IDS = new Set(["local-board"]); export function withTelemetryCompany(companyId: string | null | undefined, fn: () => T): T { - return companyId ? storage.run(companyId, fn) : fn(); + return withTelemetryActor({ companyId }, fn); +} + +export function withTelemetryActor( + scope: { companyId?: string | null; person?: TelemetryPerson | null }, + fn: () => T, +): T { + const next: TelemetryScope = {}; + if (scope.companyId) next.companyId = scope.companyId; + if (scope.person) next.person = scope.person; + return next.companyId || next.person ? storage.run(next, fn) : fn(); } export function scopedTelemetryCompanyId(): string | undefined { - return storage.getStore(); + return storage.getStore()?.companyId; +} + +export function scopedTelemetryPerson(): TelemetryPerson | undefined { + return storage.getStore()?.person; +} + +function personUserId(value: string | null | undefined): string | null { + const trimmed = value?.trim(); + return trimmed && !NON_PERSON_USER_IDS.has(trimmed) ? trimmed : null; +} + +export function resolveTelemetryPerson(input: { + taskCreatedByUserId?: string | null; + taskAssignedByUserId?: string | null; + runOwnerUserId?: string | null; +}): TelemetryPerson | null { + const creator = personUserId(input.taskCreatedByUserId); + if (creator) return { userId: creator, attributedVia: "task_creator" }; + const assigner = personUserId(input.taskAssignedByUserId); + if (assigner) return { userId: assigner, attributedVia: "task_assigner" }; + const runOwner = personUserId(input.runOwnerUserId); + if (runOwner) return { userId: runOwner, attributedVia: "run_owner" }; + return null; } diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 45b6a67d32c..0cb98887119 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -370,7 +370,7 @@ "packages/shared/src/standing-checks.ts": "7ab71e4b6984e41e1c1dfd1bddf0116041d5abf1", "packages/shared/src/subscriptions.ts": "ef70e63395ca967bbc30776b71560e96eb354a75", "packages/shared/src/summary-slot.test.ts": "5f8e222dea425f0d5bff0adb5c683087bd7cf7fb", - "packages/shared/src/telemetry/README.md": "eaa75efee24352f97ddc7d72b2e99be8b39a241b", + "packages/shared/src/telemetry/README.md": "99c1d1eddf1e5133bed005fcdef6dbfe12d83460", "packages/shared/src/telemetry/client-types.test.ts": "3b630edda4972b929ba4decd7457f226eb9cfc5b", "packages/shared/src/telemetry/client.test.ts": "5a95f2305a393c5b2e7750b10e0ed65da0f30cf8", "packages/shared/src/telemetry/client.ts": "d6534ce8aaa6f69fb2dd03e12487db024e250a96", diff --git a/ui/src/components/OnboardingWizard.test.tsx b/ui/src/components/OnboardingWizard.test.tsx index bc596d08f45..d6414c9b004 100644 --- a/ui/src/components/OnboardingWizard.test.tsx +++ b/ui/src/components/OnboardingWizard.test.tsx @@ -208,6 +208,11 @@ vi.mock("../api/approvals", () => ({ approvalsApi: mockApprovalsApi })); vi.mock("../api/secrets", () => ({ secretsApi: mockSecretsApi })); vi.mock("../api/issues", () => ({ issuesApi: mockIssuesApi })); vi.mock("../api/agentChats", () => ({ agentChatsApi: mockAgentChatsApi })); +const mockCaptureProductEvent = vi.hoisted(() => vi.fn()); +vi.mock("../lib/analytics/product-analytics", async (importOriginal) => ({ + ...(await importOriginal()), + captureProductAnalyticsEvent: mockCaptureProductEvent, +})); vi.mock("../api/projects", () => ({ projectsApi: mockProjectsApi })); vi.mock("../api/environments", () => ({ environmentsApi: mockEnvironmentsApi })); vi.mock("../api/instanceSettings", () => ({ instanceSettingsApi: mockInstanceSettingsApi })); @@ -1465,6 +1470,26 @@ describe("OnboardingWizard restore-gate (stale localStorage across accounts)", ( } finally { await act(async () => root.unmount()); } }); + it("keeps onboarding analytics on the new company when the old launch settles late", async () => { + let settleLaunch!: (value: { id: string }) => void; + mockAgentChatsApi.startOnboarding.mockImplementationOnce(() => new Promise((resolve) => { settleLaunch = resolve; })); + const { root, queryClient } = await openSavedReview("/SC/onboarding"); + try { + await act(async () => [...document.querySelectorAll("button")].find(button => button.textContent?.trim() === "Get started")!.click()); + mockRouterState.pathname = "/OTHER/onboarding"; + await renderWizardView(root, queryClient); + for (let tick = 0; tick < 8; tick++) await flushReact(); + await act(async () => settleLaunch({ id: "onboarding-chat" })); + for (let tick = 0; tick < 4; tick++) await flushReact(); + expect(mockNavigate).not.toHaveBeenCalled(); + mockCaptureProductEvent.mockClear(); + mockRouterState.pathname = "/OTHER/dashboard"; + await renderWizardView(root, queryClient); + for (let tick = 0; tick < 4; tick++) await flushReact(); + expect(mockCaptureProductEvent).toHaveBeenCalledWith("onboarding_dismissed", { step: "create_agent" }); + } finally { await act(async () => root.unmount()); } + }); + it("keeps restored review progress when initial effects replay in StrictMode", async () => { const { root, draft } = await openSavedReview("/SC/onboarding", "auto", { strict: true }); try { diff --git a/ui/src/components/OnboardingWizard.tsx b/ui/src/components/OnboardingWizard.tsx index d74842a8279..d1b3bbe2ce3 100644 --- a/ui/src/components/OnboardingWizard.tsx +++ b/ui/src/components/OnboardingWizard.tsx @@ -140,6 +140,7 @@ import { OnboardingHeading } from "./onboarding/OnboardingPrimitives"; import { DEFAULT_AGENT_ROLE } from "../lib/onboarding-agent-role"; import { capsuleHeroMotion, capsuleRoomEnter, capsuleRoomExit, heroRoomArrival, heroRoomMotion, ledeMotion, stepContentMotion, titleSwapMotion } from "./onboarding/onboarding-motion"; import { Badge } from "@/components/ui/badge"; +import { captureOnboardingStepCompleted, useOnboardingAnalytics } from "@/lib/analytics/onboarding-analytics"; import { ArrowLeft, ArrowRight, @@ -677,6 +678,11 @@ function OnboardingWizardInner({ (savedKeys.subscriptions.length > 0 || (adapterType === "claude_local" && savedKeys.storedLogin.data)) ? "subscription" : savedKeys.options.length || adapterType === "opencode_local" ? "api" : "subscription" ); + const onboardingAnalytics = useOnboardingAnalytics({ + open: effectiveOnboardingOpen, + step, + connectOutcome: credentialMode === "api" ? "api_key" : "subscription", + }); const [createdCompanyPrefix, setCreatedCompanyPrefix] = useState< string | null >((saved?.createdCompanyPrefix as string) ?? null); @@ -830,6 +836,9 @@ function OnboardingWizardInner({ setStep(initialStepRef.current); setEntryStep(initialStepRef.current); } + if (!previousRequest?.open) { + onboardingAnalytics.opened(initialStepRef.current && !resumesRouteDraft ? initialStepRef.current : step, step); + } if (routeCompanyId) { // Claim ownership only when the route *introduces* a company. A route // that merely names the one already in hand - the wizard created it, @@ -1831,6 +1840,7 @@ function OnboardingWizardInner({ } function handleClose() { + onboardingAnalytics.dismissed(step); reset(); closeOnboarding(); // On the /onboarding route the wizard is also kept open by the route @@ -1884,9 +1894,12 @@ function OnboardingWizardInner({ } setLoading(true); setError(null); + captureOnboardingStepCompleted("review"); try { await agentChatsApi.startOnboarding(createdCompanyId, createdAgentId); + captureOnboardingStepCompleted("launch", "succeeded"); if (!stillTheSameCompany(createdCompanyId)) return; + onboardingAnalytics.finished(); const prefix = createdCompanyPrefix; const agentId = createdAgentId; // Route sync preserves this explicit destination over page memory. @@ -1895,6 +1908,7 @@ function OnboardingWizardInner({ closeOnboarding(); navigate(`${prefix ? `/${prefix}` : ""}/chats/${agentId}`); } catch (err) { + captureOnboardingStepCompleted("launch", "failed"); setError(err instanceof Error ? err.message : "Failed to open your chat. Please try again."); } finally { setLoading(false); diff --git a/ui/src/lib/analytics/onboarding-analytics.test.tsx b/ui/src/lib/analytics/onboarding-analytics.test.tsx new file mode 100644 index 00000000000..37abe17685c --- /dev/null +++ b/ui/src/lib/analytics/onboarding-analytics.test.tsx @@ -0,0 +1,260 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { BrowserAnalyticsConfig } from "@paperclipai/shared"; +import { + captureProductAnalyticsEvent, + resetProductAnalyticsForTest, + setProductAnalyticsUser, + syncProductAnalytics, + type ProductAnalyticsClient, +} from "./product-analytics"; +import { captureOnboardingStepCompleted, useOnboardingAnalytics } from "./onboarding-analytics"; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const unmounts: Array<() => void> = []; + +function renderHook(callback: (props: P) => R, options: { initialProps: P }) { + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + let props = options.initialProps; + const result = {} as { current: R }; + function Harness() { + result.current = callback(props); + return null; + } + function rerender(next: P = props) { + props = next; + act(() => root.render(createElement(Harness))); + } + unmounts.push(() => { + act(() => root.unmount()); + host.remove(); + }); + rerender(); + return { result, rerender }; +} + +const config: BrowserAnalyticsConfig = { + token: "phc_test", + apiHost: "/_ph", + uiHost: "https://us.posthog.com", + environment: "staging", + stackId: "stack-1", + replayEnabled: false, +}; + +function mockClient() { + return { + init: vi.fn(), + register: vi.fn(), + identify: vi.fn(), + group: vi.fn(), + reset: vi.fn(), + capture: vi.fn(), + opt_out_capturing: vi.fn(), + opt_in_capturing: vi.fn(), + }; +} + +async function flush() { + await Promise.resolve(); + await Promise.resolve(); +} + +let client: ReturnType; +let scheduled: (() => void) | null; +const load = () => Promise.resolve(client as unknown as ProductAnalyticsClient); +const schedule = (start: () => void) => { + scheduled = start; + return () => { scheduled = null; }; +}; + +async function startAnalytics() { + syncProductAnalytics(config, "granted", { load, schedule, immediate: true }); + await flush(); +} + +function captured() { + return client.capture.mock.calls.map(([event, properties]) => [event, properties]); +} + +beforeEach(() => { + client = mockClient(); + scheduled = null; +}); + +afterEach(() => { + for (const unmount of unmounts.splice(0)) unmount(); + resetProductAnalyticsForTest(); +}); + +describe("browser product events", () => { + it("captures nothing when browser analytics is off", async () => { + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + await flush(); + expect(client.capture).not.toHaveBeenCalled(); + }); + + it.each(["denied", "ask"] as const)("captures nothing when consent is %s", async (consent) => { + syncProductAnalytics(config, consent, { load, schedule, immediate: true }); + await flush(); + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + await flush(); + expect(client.init).not.toHaveBeenCalled(); + expect(client.capture).not.toHaveBeenCalled(); + }); + + it("holds events from before posthog-js loads and sends them with their own time", async () => { + syncProductAnalytics(config, "granted", { load, schedule }); + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + expect(client.capture).not.toHaveBeenCalled(); + scheduled!(); + await flush(); + expect(client.capture).toHaveBeenCalledTimes(1); + expect(client.capture).toHaveBeenCalledWith( + "onboarding_opened", + { step: "create_agent" }, + { timestamp: expect.any(Date) }, + ); + }); + + it("drops held events when consent is withdrawn before posthog-js loads", async () => { + syncProductAnalytics(config, "granted", { load, schedule }); + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + scheduled!(); + syncProductAnalytics(config, "denied", { load, schedule }); + await flush(); + captureProductAnalyticsEvent("onboarding_dismissed", { step: "create_agent" }); + expect(client.capture).not.toHaveBeenCalled(); + }); + + it("drops held events when the signed-in person changes before posthog-js loads", async () => { + setProductAnalyticsUser({ id: "user-1", email: null, name: null }); + syncProductAnalytics(config, "granted", { load, schedule }); + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + setProductAnalyticsUser(null); + setProductAnalyticsUser({ id: "user-2", email: null, name: null }); + captureProductAnalyticsEvent("onboarding_opened", { step: "connect_model" }); + scheduled!(); + await flush(); + expect(captured()).toEqual([["onboarding_opened", { step: "connect_model" }]]); + expect(client.identify).toHaveBeenCalledWith("user-2", {}); + }); + + it("keeps held events when the same person is set again before posthog-js loads", async () => { + setProductAnalyticsUser({ id: "user-1", email: null, name: null }); + syncProductAnalytics(config, "granted", { load, schedule }); + captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" }); + setProductAnalyticsUser({ id: "user-1", email: "jane@example.com", name: null }); + scheduled!(); + await flush(); + expect(captured()).toEqual([["onboarding_opened", { step: "create_agent" }]]); + }); + + it("stops capturing after consent is withdrawn", async () => { + await startAnalytics(); + syncProductAnalytics(config, "denied", { load, schedule }); + captureProductAnalyticsEvent("onboarding_opened", {}); + expect(client.capture).not.toHaveBeenCalled(); + }); +}); + +describe("onboarding funnel events", () => { + function renderWizard(initial: { open: boolean; step: number }) { + return renderHook( + (props: { open: boolean; step: number; connectOutcome: "subscription" | "api_key" }) => useOnboardingAnalytics(props), + { initialProps: { ...initial, connectOutcome: "subscription" as "subscription" | "api_key" } }, + ); + } + + it("captures each transition once through the wizard", async () => { + await startAnalytics(); + const wizard = renderWizard({ open: false, step: 1 }); + wizard.rerender({ open: true, step: 1, connectOutcome: "subscription" }); + act(() => wizard.result.current.opened(3, 1)); + wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" }); + wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" }); + wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" }); + wizard.rerender({ open: true, step: 4, connectOutcome: "api_key" }); + wizard.rerender({ open: true, step: 5, connectOutcome: "api_key" }); + captureOnboardingStepCompleted("review"); + captureOnboardingStepCompleted("launch", "succeeded"); + act(() => wizard.result.current.finished()); + wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" }); + + expect(captured()).toEqual([ + ["onboarding_opened", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "create_agent" }], + ["onboarding_step_completed", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "connect_model" }], + ["onboarding_step_completed", { step: "connect_model", outcome: "api_key" }], + ["onboarding_step_viewed", { step: "review" }], + ["onboarding_step_completed", { step: "review" }], + ["onboarding_step_completed", { step: "launch", outcome: "succeeded" }], + ]); + }); + + it("does not count a stale step from the opening render as a view", async () => { + await startAnalytics(); + const wizard = renderWizard({ open: true, step: 5 }); + act(() => wizard.result.current.opened(3, 5)); + wizard.rerender({ open: true, step: 5, connectOutcome: "subscription" }); + wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" }); + + expect(captured()).toEqual([ + ["onboarding_opened", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "create_agent" }], + ]); + }); + + it("captures a dismissal once and a reopen as a new session", async () => { + await startAnalytics(); + const wizard = renderWizard({ open: true, step: 4 }); + act(() => wizard.result.current.opened(4, 4)); + act(() => wizard.result.current.opened(4, 4)); + act(() => wizard.result.current.dismissed(4)); + act(() => wizard.result.current.dismissed(4)); + wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" }); + wizard.rerender({ open: true, step: 1, connectOutcome: "subscription" }); + act(() => wizard.result.current.opened(3, 1)); + + expect(captured()).toEqual([ + ["onboarding_opened", { step: "connect_model" }], + ["onboarding_step_viewed", { step: "connect_model" }], + ["onboarding_dismissed", { step: "connect_model" }], + ["onboarding_opened", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "create_agent" }], + ]); + }); + + it("captures a dismissal when the wizard closes before launch", async () => { + await startAnalytics(); + const wizard = renderWizard({ open: true, step: 3 }); + act(() => wizard.result.current.opened(3, 3)); + wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" }); + wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" }); + wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" }); + + expect(captured()).toEqual([ + ["onboarding_opened", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "create_agent" }], + ["onboarding_step_completed", { step: "create_agent" }], + ["onboarding_step_viewed", { step: "connect_model" }], + ["onboarding_dismissed", { step: "connect_model" }], + ]); + }); + + it("captures nothing in the wizard when consent is not granted", async () => { + syncProductAnalytics(config, "ask", { load, schedule, immediate: true }); + const wizard = renderWizard({ open: true, step: 3 }); + act(() => wizard.result.current.opened(3, 3)); + wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" }); + act(() => wizard.result.current.dismissed(4)); + expect(client.capture).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/src/lib/analytics/onboarding-analytics.ts b/ui/src/lib/analytics/onboarding-analytics.ts new file mode 100644 index 00000000000..271a732aaff --- /dev/null +++ b/ui/src/lib/analytics/onboarding-analytics.ts @@ -0,0 +1,92 @@ +import { useCallback, useEffect, useRef } from "react"; +import { captureProductAnalyticsEvent } from "./product-analytics"; + +export const ONBOARDING_ANALYTICS_EVENTS = { + opened: "onboarding_opened", + stepViewed: "onboarding_step_viewed", + stepCompleted: "onboarding_step_completed", + dismissed: "onboarding_dismissed", +} as const; + +export type OnboardingAnalyticsStep = "create_agent" | "connect_model" | "review" | "launch"; +export type OnboardingAnalyticsOutcome = "subscription" | "api_key" | "succeeded" | "failed"; + +const WIZARD_STEPS: Record = { + 3: "create_agent", + 4: "connect_model", + 5: "review", +}; + +export function onboardingAnalyticsStepFor(wizardStep: number): OnboardingAnalyticsStep | null { + return WIZARD_STEPS[wizardStep] ?? null; +} + +function stepProperties(step: OnboardingAnalyticsStep | null, outcome?: OnboardingAnalyticsOutcome) { + const properties: Record = {}; + if (step) properties.step = step; + if (outcome) properties.outcome = outcome; + return properties; +} + +export function captureOnboardingStepCompleted(step: OnboardingAnalyticsStep, outcome?: OnboardingAnalyticsOutcome) { + captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepCompleted, stepProperties(step, outcome)); +} + +export function useOnboardingAnalytics(input: { + open: boolean; + step: number; + connectOutcome: OnboardingAnalyticsOutcome; +}) { + const session = useRef<{ pendingStep: number | null; viewedStep: number } | null>(null); + const connectOutcome = useRef(input.connectOutcome); + connectOutcome.current = input.connectOutcome; + + const opened = useCallback((entryStep: number, renderedStep: number) => { + if (session.current) return; + session.current = { pendingStep: entryStep === renderedStep ? null : entryStep, viewedStep: entryStep }; + const step = onboardingAnalyticsStepFor(entryStep); + captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.opened, stepProperties(step)); + if (step) captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepViewed, stepProperties(step)); + }, []); + + const dismissed = useCallback((step: number) => { + if (!session.current) return; + session.current = null; + captureProductAnalyticsEvent( + ONBOARDING_ANALYTICS_EVENTS.dismissed, + stepProperties(onboardingAnalyticsStepFor(step)), + ); + }, []); + + const finished = useCallback(() => { + session.current = null; + }, []); + + const { open, step } = input; + useEffect(() => { + const current = session.current; + if (!open) { + if (current) dismissed(current.viewedStep); + return; + } + if (!current) return; + if (current.pendingStep !== null) { + if (step !== current.pendingStep) return; + current.pendingStep = null; + } + const previous = current.viewedStep; + if (previous === step) return; + current.viewedStep = step; + const previousStep = onboardingAnalyticsStepFor(previous); + if (previousStep && step === previous + 1) { + captureOnboardingStepCompleted( + previousStep, + previousStep === "connect_model" ? connectOutcome.current : undefined, + ); + } + const viewed = onboardingAnalyticsStepFor(step); + if (viewed) captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepViewed, stepProperties(viewed)); + }, [open, step]); + + return { opened, dismissed, finished }; +} diff --git a/ui/src/lib/analytics/product-analytics.ts b/ui/src/lib/analytics/product-analytics.ts index 4468de6765b..16f89d2fd87 100644 --- a/ui/src/lib/analytics/product-analytics.ts +++ b/ui/src/lib/analytics/product-analytics.ts @@ -37,6 +37,7 @@ export interface ProductAnalyticsClient { identify(distinctId: string, personProperties?: Record): void; group(groupType: string, groupKey: string): void; reset(): void; + capture?(event: string, properties?: Record, options?: { timestamp?: Date }): unknown; opt_out_capturing?(): void; opt_in_capturing?(): void; has_opted_out_capturing?(): boolean; @@ -210,8 +211,17 @@ type State = { pendingClient: ProductAnalyticsClient | null; loading: boolean; optedOut: boolean; + queued: QueuedProductEvent[]; }; +type QueuedProductEvent = { + event: string; + properties: Record; + timestamp: Date; +}; + +const MAX_QUEUED_PRODUCT_EVENTS = 50; + let state: State | null = null; let wantedUser: ProductAnalyticsUser | null = null; let identifiedUserId: string | null = null; @@ -236,6 +246,21 @@ function applyIdentity(client: ProductAnalyticsClient, config: BrowserAnalyticsC } } +function sendProductEvent(client: ProductAnalyticsClient, queued: QueuedProductEvent) { + client.capture?.(queued.event, queued.properties, { timestamp: queued.timestamp }); +} + +export function captureProductAnalyticsEvent(event: string, properties: Record = {}): void { + const current = state; + if (!current || current.optedOut) return; + const queued = { event, properties: { ...properties }, timestamp: new Date() }; + if (current.client) { + sendProductEvent(current.client, queued); + return; + } + if (current.queued.length < MAX_QUEUED_PRODUCT_EVENTS) current.queued.push(queued); +} + function optOut(client: ProductAnalyticsClient) { client.opt_out_capturing?.(); client.stopSessionRecording?.(); @@ -253,6 +278,9 @@ function initialize(current: State, client: ProductAnalyticsClient) { client.register(superProperties(config)); current.client = client; applyIdentity(client, config); + const queued = current.queued; + current.queued = []; + for (const event of queued) sendProductEvent(client, event); } /** @@ -283,6 +311,7 @@ export function startProductAnalytics( cancelSchedule: null, loading: false, optedOut: false, + queued: [], }; state = current; const load = options.load ?? loadPostHog; @@ -309,6 +338,7 @@ export function startProductAnalytics( /** Set (or clear, on sign-out) the signed-in user. Safe to call before posthog-js loads. */ export function setProductAnalyticsUser(user: ProductAnalyticsUser | null): void { + if (wantedUser && wantedUser.id !== user?.id && state) state.queued = []; wantedUser = user; if (state?.client) applyIdentity(state.client, state.config); } @@ -341,6 +371,7 @@ function stopProductAnalytics(): void { return; } state.optedOut = true; + state.queued = []; if (state.client) optOut(state.client); }