feat(analytics): attribute agent work and onboarding steps to a person on Cloud (#22260)

## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work
> - Paperclip Cloud sends product events straight to PostHog: server
events from `server/src/services/posthog-direct/` and browser events
from `ui/src/lib/analytics/`
> - A person-level funnel goes from sign-up to the first finished task
> - Agent-driven events (`agent.task_run`, `agent.task_completed`,
`agent.first_heartbeat`) used `workspace:<stackId>` as the distinct id,
so the funnel broke at those steps
> - The in-workspace onboarding wizard sent no step events, so we could
not see where people stop before their first agent
> - This pull request attributes agent work to the responsible person on
the Cloud direct path, and adds consent-gated onboarding step events in
the browser
> - The benefit is one PostHog person per customer across sign-up,
onboarding, agent runs, and task completion. Self-hosted output does not
change.

## Linked Issues or Issue Description

**Subsystem affected**
Cloud product analytics: Cloud direct PostHog delivery (server) and
browser product analytics (UI onboarding wizard).

**Problem or motivation**
A PostHog funnel from sign-up to the first finished task cannot follow
one person. Agent-driven events run in the background, with no signed-in
request, so they use the workspace id as the distinct id. The onboarding
wizard also sends no events, so we cannot see which step people leave.

**Proposed solution**
Give background emit sites a narrow scope that names the person
responsible for the work. Use the task creator first, then the task's
responsible user, then the run owner. Keep the signed-in request user
first. Add `attributed_via` to every Cloud direct event. Add four
browser events to the onboarding wizard. Send them through the existing
analytics module, so the consent rules stay the same.

**Alternatives considered**
A lookup in the transport at send time. We did not use it: it adds
database reads to the telemetry path and loses the emit site's context.
A new contract event for onboarding steps. We did not use it: the server
does not see the wizard steps, and the UI must never send
`PAPERCLIP_EVENTS` names.

**Additional context**
Self-hosted installs keep the collector path. The collector envelope
stays byte-identical (a test checks it).

## What Changed

- `server/src/telemetry-company-scope.ts`: add `withTelemetryActor({
companyId, person }, fn)`, `scopedTelemetryPerson()`, and
`resolveTelemetryPerson()`. The resolver picks the task creator, then
the task's responsible user, then the run owner. It ignores
`local-board`. `withTelemetryCompany` now calls `withTelemetryActor`.
- `server/src/services/posthog-direct/request-context.ts`: a signed-in
request user wins (`attributed_via: request`). Without one, the scoped
person is used.
- `server/src/services/posthog-direct/event-shape.ts`: every Cloud
direct event (including `cloud.workspace_started`) has `attributed_via`
(`request`, `task_creator`, `task_assigner`, `run_owner`, or `none`). An
event with a person keeps today's person-event shape (no
`$process_person_profile: false`).
- `server/src/services/agent-task-run-telemetry.ts`: `agent.task_run`
reads the task creator and responsible user in the existing agent query
(one left join, no new query), and the run's `responsibleUserId`.
- `server/src/routes/issues.ts`: `agent.task_completed` uses the issue
row that the route already has, and the agent actor's `onBehalfOfUserId`
as the run owner.
- `server/src/services/heartbeat/agent-status.ts`:
`agent.first_heartbeat` uses the agent's `createdByUserId` (already
loaded) as the run owner.
- `ui/src/lib/analytics/product-analytics.ts`: add
`captureProductAnalyticsEvent()`. It captures only with a browser
analytics config and granted consent. Events from before posthog-js
loads wait in a queue (at most 50) and keep their time. Withdrawn
consent or a change of signed-in person drops the queue.
- `ui/src/lib/analytics/onboarding-analytics.ts` and
`ui/src/components/OnboardingWizard.tsx`: capture `onboarding_opened`,
`onboarding_step_viewed`, `onboarding_step_completed` (with `step` and,
for `connect_model` and `launch`, `outcome`), and
`onboarding_dismissed`. `step` is `create_agent`, `connect_model`,
`review`, or `launch`. No event has free text.
- Docs: `packages/shared/src/telemetry/README.md` (Cloud Direct
Delivery) and `doc/cloud-browser-analytics.md` (new "Onboarding events"
section).

## Verification

- `server/src/__tests__/posthog-direct-telemetry.test.ts`: attribution
matrix (creator, assigner, run owner, none stays on
`workspace:<stackId>` with `$process_person_profile: false`), the
request user wins over the scope, and the self-hosted collector envelope
is byte-identical with a person scope.
- `server/src/__tests__/agent-task-run-telemetry.test.ts` (embedded
Postgres): `agent.task_run` scope for creator, assigner, run owner, and
nobody; another company's task is never read.
- `server/src/__tests__/issue-telemetry-routes.test.ts`:
`agent.task_completed` scope for creator and assigner.
- `server/src/__tests__/heartbeat-process-recovery.test.ts`:
`agent.first_heartbeat` scope is the agent owner.
- `ui/src/components/OnboardingWizard.test.tsx`: a launch for the old
company that settles after a switch to another company does not end the
new analytics session.
- `ui/src/lib/analytics/onboarding-analytics.test.tsx`: each transition
fires once, held events drop on a person change, a stale opening step is
not a view, dismissal fires once, nothing is captured when analytics is
off or consent is `denied` or `ask`, and queued events keep their time.
- `ui/src/lib/analytics/product-analytics-events.test.ts` still passes
(the UI never captures a `PAPERCLIP_EVENTS` name).
- Local Cloud-mode run: an isolated instance (own Postgres, trusted
Cloud tenant headers, `PAPERCLIP_POSTHOG_HOST` set to a local capture
server, never real PostHog). A stub stood in for posthog-js and sent
through the `/_ph` proxy.
- A user created a task and assigned it to the agent. `agent.task_run`
arrived with `distinct_id: paperclip-id:local-user-1`, `attributed_via:
task_creator`, the raw `task_id`, and no `$process_person_profile`. (The
run itself failed locally because the Cloud runner is not available. The
event still shows the attribution.)
- `agent.first_heartbeat` arrived with the same person and
`attributed_via: run_owner`. `server.started` kept
`workspace:stack-local` with `attributed_via: none`.
- Headless Chromium walked the wizard: `onboarding_opened`
(create_agent), `onboarding_step_viewed` (create_agent),
`onboarding_step_completed` (create_agent), `onboarding_step_viewed`
(connect_model), `onboarding_step_completed` (connect_model, `api_key`),
`onboarding_step_viewed` (review), `onboarding_step_completed` (review),
`onboarding_step_completed` (launch, `succeeded`). Each fired once, with
the signed-in user as the distinct id. Events from before posthog-js
loaded arrived with their original time.
- A second walk left the onboarding page on the connect step and sent
one `onboarding_dismissed` (connect_model).

## Risks

- Low risk for self-hosted: the collector path never reads the new
scope, and a test checks the envelope bytes.
- Cloud: agent events move from the workspace person to a real person.
Insights that count `workspace:` distinct ids for these events will
change. `attributed_via` lets a query filter or split them.
- `task_assigner` uses the task's responsible user. Paperclip does not
store who assigned a task, so this is the person who owns or delegated
the task.
- Privacy: the new data is Cloud-only and uses ids that Cloud events
already carry. No new free text, email, or name goes on an event.
**Privacy review requested** for the Cloud attribution change (per
`rules/data-paths.mdc`). The generated contract does not change:
`attributed_via` is a Cloud-only property, like `stack_id` and
`user_id`.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude
Code with tool use (shell, file edits, headless browser).

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 authored and GitHub committed 2026-11-18 18:44:45 -06:00
1 parent e2cee049f9
commit 92dbf39e6f
19 files changed
+766 -22

No files matched your search

+20
View File
@@ -233,6 +233,26 @@ This value is used only for location, never for limits.
- API requests to our own host carry PostHog tracing headers, so server events - API requests to our own host carry PostHog tracing headers, so server events
link to the browser session. link to the browser session.
## Onboarding events
The in-workspace onboarding wizard (`ui/src/components/OnboardingWizard.tsx`)
captures four browser events through `ui/src/lib/analytics/onboarding-analytics.ts`.
They follow the same consent rules as every other browser event: without a
browser analytics config or with consent not granted, nothing is captured.
Events from before posthog-js loads wait in a short queue (at most 50) and keep
their original time. Withdrawing consent or a change of signed-in person drops
the queue.
| Event | When | Properties |
| --- | --- | --- |
| `onboarding_opened` | The wizard opens, once per open. Auto-open (`useAgentlessOnboardingAutoOpen`), a button, and the `/onboarding` route all count. | `step` when the wizard opens on an agent step |
| `onboarding_step_viewed` | A step is shown, once per step change. | `step` |
| `onboarding_step_completed` | The customer moves forward one step, presses Launch on the review step, or Launch finishes. | `step`; `outcome` for `connect_model` (`subscription` or `api_key`) and for `launch` (`succeeded` or `failed`) |
| `onboarding_dismissed` | The wizard closes before Launch finishes, for example when the customer leaves the onboarding page. | `step` when closed on an agent step |
`step` is one of `create_agent`, `connect_model`, `review`, or `launch`. The
organization-naming steps have no `step` value. No event carries free text.
## Privacy ## Privacy
Session replay shows text by default, the way PostHog normally works. Text is Session replay shows text by default, the way PostHog normally works. Text is
+13
View File
@@ -150,6 +150,19 @@ exposes the `TelemetryTransport` seam on `TelemetryClient`:
reports on. Only the direct transport reads this scope, and it uses it only reports on. Only the direct transport reads this scope, and it uses it only
when no live request supplies a company. The collector envelope and the when no live request supplies a company. The collector envelope and the
dimensions do not change. dimensions do not change.
- Agent-driven events (`agent.task_run`, `agent.task_completed`, and
`agent.first_heartbeat`) also name the person responsible for the work, with
`withTelemetryActor({ companyId, person }, fn)` and
`resolveTelemetryPerson()` from the same file. The person is the user who
created the task, else the task's responsible user (who assigned or
delegated it), else the user who owns the run (for `agent.first_heartbeat`,
the user who created the agent). The emit site reads these ids from rows it
already loads; the transport does no database reads. A signed-in user on a
live request always wins over this scope. With no person, the event stays on
`workspace:<stackId>` with `$process_person_profile: false`. Every direct
event carries `attributed_via`: `request`, `task_creator`, `task_assigner`,
`run_owner`, or `none`. The collector path never reads this scope, so the
self-hosted envelope does not change.
Cloud is Paperclip's own service under the Cloud terms, so its events carry Cloud is Paperclip's own service under the Cloud terms, so its events carry
real identifiers: raw agent and task ids, the stack id, and the company and real identifiers: raw agent and task ids, the stack id, and the company and
@@ -1,12 +1,12 @@
import { randomBytes, randomUUID } from "node:crypto"; import { randomBytes, randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; import { agents, companies, createDb, heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { eq } from "drizzle-orm"; import { eq } from "drizzle-orm";
import { import {
getEmbeddedPostgresTestSupport, getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase, startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js"; } from "./helpers/embedded-postgres.js";
import { scopedTelemetryCompanyId } from "../telemetry-company-scope.js"; import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../telemetry-company-scope.js";
const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() }));
const mockTrackAgentTaskRun = vi.hoisted(() => vi.fn()); const mockTrackAgentTaskRun = vi.hoisted(() => vi.fn());
@@ -44,6 +44,7 @@ describeEmbeddedPostgres("emitAgentTaskRun", () => {
afterEach(async () => { afterEach(async () => {
vi.clearAllMocks(); vi.clearAllMocks();
await db.delete(heartbeatRuns); await db.delete(heartbeatRuns);
await db.delete(issues);
await db.delete(agents); await db.delete(agents);
await db.delete(companies); await db.delete(companies);
}); });
@@ -120,6 +121,45 @@ describeEmbeddedPostgres("emitAgentTaskRun", () => {
expect(scopedTelemetryCompanyId()).toBeUndefined(); expect(scopedTelemetryCompanyId()).toBeUndefined();
}); });
it.each([
{ name: "task creator", issue: { createdByUserId: "user-creator", responsibleUserId: "user-assigner" }, runOwner: "user-owner", expected: { userId: "user-creator", attributedVia: "task_creator" } },
{ name: "task assigner", issue: { createdByUserId: null, responsibleUserId: "user-assigner" }, runOwner: "user-owner", expected: { userId: "user-assigner", attributedVia: "task_assigner" } },
{ name: "run owner", issue: { createdByUserId: null, responsibleUserId: null }, runOwner: "user-owner", expected: { userId: "user-owner", attributedVia: "run_owner" } },
{ name: "nobody", issue: { createdByUserId: null, responsibleUserId: null }, runOwner: null, expected: undefined },
])("tracks agent.task_run for the person responsible: $name", async ({ issue, runOwner, expected }) => {
await seedCompanyAndAgent();
const issueId = randomUUID();
await db.insert(issues).values({ id: issueId, companyId, title: "Write the report", assigneeAgentId: agentId, createdByAgentId: issue.createdByUserId ? null : agentId, ...issue });
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded",
responsibleUserId: runOwner, contextSnapshot: { issueId } }).returning();
const people: unknown[] = [];
const recordPerson = () => people.push(scopedTelemetryPerson());
mockTrackAgentTaskRun.mockImplementationOnce(recordPerson).mockImplementationOnce(recordPerson);
await emitAgentTaskRun(db, run);
await emitAgentTaskRunById(db, { runId: run.id, companyId });
expect(people).toEqual([expected, expected]);
expect(mockTrackAgentTaskRun).toHaveBeenCalledWith(mockTelemetryClient, expect.objectContaining({ taskId: issueId }));
});
it("does not read another company's task for attribution", async () => {
await seedCompanyAndAgent();
const otherCompanyId = randomUUID();
await db.insert(companies).values({ id: otherCompanyId, name: "Other", issuePrefix: `O${otherCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false });
const issueId = randomUUID();
await db.insert(issues).values({ id: issueId, companyId: otherCompanyId, title: "Other task", createdByUserId: "user-other" });
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", contextSnapshot: { issueId } }).returning();
const people: unknown[] = [];
mockTrackAgentTaskRun.mockImplementationOnce(() => people.push(scopedTelemetryPerson()));
await emitAgentTaskRun(db, run);
expect(people).toEqual([undefined]);
await db.delete(issues);
await db.delete(companies).where(eq(companies.id, otherCompanyId));
});
it("emits the same event from a small scoped run read without loading execution payloads", async () => { it("emits the same event from a small scoped run read without loading execution payloads", async () => {
await seedCompanyAndAgent(); await seedCompanyAndAgent();
const payload = randomBytes(128 * 1024).toString("base64"); const payload = randomBytes(128 * 1024).toString("base64");
@@ -124,7 +124,7 @@ import {
getEmbeddedPostgresTestSupport, getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase, startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js"; } from "./helpers/embedded-postgres.js";
import { scopedTelemetryCompanyId } from "../telemetry-company-scope.js"; import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../telemetry-company-scope.js";
import type { EnvironmentLease } from "@paperclipai/shared"; import type { EnvironmentLease } from "@paperclipai/shared";
import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js"; import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js";
import { runningProcesses, type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts"; import { runningProcesses, type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts";
@@ -10040,9 +10040,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
agentStatus: "running", agentStatus: "running",
includeIssue: false, includeIssue: false,
}); });
await db.update(agents).set({ createdByUserId: "paperclip-id:agent-owner" }).where(eq(agents.id, agentId));
const heartbeat = heartbeatService(db); const heartbeat = heartbeatService(db);
const scopedCompanies: Array<string | undefined> = []; const scopedCompanies: Array<string | undefined> = [];
mockTrackAgentFirstHeartbeat.mockImplementationOnce(() => scopedCompanies.push(scopedTelemetryCompanyId())); const scopedPeople: unknown[] = [];
mockTrackAgentFirstHeartbeat.mockImplementationOnce(() => {
scopedCompanies.push(scopedTelemetryCompanyId());
scopedPeople.push(scopedTelemetryPerson());
});
await heartbeat.cancelRun(runId); await heartbeat.cancelRun(runId);
@@ -10054,6 +10059,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}), }),
); );
expect(scopedCompanies).toEqual([companyId]); expect(scopedCompanies).toEqual([companyId]);
expect(scopedPeople).toEqual([{ userId: "paperclip-id:agent-owner", attributedVia: "run_owner" }]);
}); });
it("preserves first-heartbeat telemetry after a timer interval claim", async () => { it("preserves first-heartbeat telemetry after a timer interval claim", async () => {
@@ -219,6 +219,40 @@ describe("issue telemetry routes", () => {
expect(scopedCompanies).toEqual(["company-1"]); expect(scopedCompanies).toEqual(["company-1"]);
}, 10_000); }, 10_000);
it.each([
{ name: "task creator", issue: { createdByUserId: "paperclip-id:creator", responsibleUserId: "paperclip-id:assigner" }, expected: { userId: "paperclip-id:creator", attributedVia: "task_creator" } },
{ name: "task assigner", issue: { createdByUserId: null, responsibleUserId: "paperclip-id:assigner" }, expected: { userId: "paperclip-id:assigner", attributedVia: "task_assigner" } },
])("attributes agent task-completed telemetry to the $name", async ({ issue, expected }) => {
const { scopedTelemetryPerson } = await import("../telemetry-company-scope.js");
const people: unknown[] = [];
mockTrackAgentTaskCompleted.mockImplementationOnce(() => people.push(scopedTelemetryPerson()));
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
...makeIssue("todo"),
...issue,
...patch,
}));
mockAgentService.getById.mockResolvedValue({
id: "agent-1",
companyId: "company-1",
role: "engineer",
adapterType: "codex_local",
adapterConfig: {},
});
const app = await createApp({
type: "agent",
agentId: "agent-1",
companyId: "company-1",
runId: "22222222-2222-4222-8222-222222222222",
});
const res = await request(app)
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
.send({ status: "done" });
expect(res.status).toBe(200);
await vi.waitFor(() => expect(people).toEqual([expected]));
}, 10_000);
it("does not emit agent task-completed telemetry for board-driven completions", async () => { it("does not emit agent task-completed telemetry for board-driven completions", async () => {
const app = await createApp({ const app = await createApp({
type: "board", type: "board",
@@ -59,6 +59,7 @@ describe("Cloud direct PostHog event shape", () => {
const intended = { const intended = {
distinct_id: "workspace:stack-123", distinct_id: "workspace:stack-123",
stack_id: "stack-123", stack_id: "stack-123",
attributed_via: "none",
environment: "staging", environment: "staging",
$groups: { workspace: "stack-123" }, $groups: { workspace: "stack-123" },
...("agent_id" in inputDims ? { agent_id: inputDims.agent_id } : {}), ...("agent_id" in inputDims ? { agent_id: inputDims.agent_id } : {}),
@@ -85,6 +86,7 @@ describe("Cloud direct PostHog event shape", () => {
distinct_id: "user-1", distinct_id: "user-1",
user_id: "user-1", user_id: "user-1",
company_id: "company-1", company_id: "company-1",
attributed_via: "request",
$session_id: "sess-1", $session_id: "sess-1",
$window_id: "win-1", $window_id: "win-1",
$groups: { workspace: "stack-123" }, $groups: { workspace: "stack-123" },
@@ -31,7 +31,7 @@ import {
capturePostHogEventContext, capturePostHogEventContext,
postHogRequestContextMiddlewareForEnv, postHogRequestContextMiddlewareForEnv,
} from "../services/posthog-direct/request-context.js"; } from "../services/posthog-direct/request-context.js";
import { withTelemetryCompany } from "../telemetry-company-scope.js"; import { resolveTelemetryPerson, withTelemetryActor, withTelemetryCompany } from "../telemetry-company-scope.js";
import { serverVersion } from "../version.js"; import { serverVersion } from "../version.js";
const COLLECTOR = "http://collector.test/ingest"; const COLLECTOR = "http://collector.test/ingest";
@@ -159,6 +159,40 @@ describe("server telemetry sender selection", () => {
expect(body).not.toContain("company"); expect(body).not.toContain("company");
}); });
it("self-hosted: a background person scope leaves the collector envelope byte-identical", async () => {
const client = initTelemetry({ enabled: true }, { env: {} })!;
const person = resolveTelemetryPerson({ taskCreatedByUserId: "user-creator" });
await withTelemetryActor({ companyId: "company-bg", person }, async () => {
await Promise.resolve();
trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" });
});
await shutdownTelemetry();
expect(hoisted.posthogNodeLoads).toBe(0);
expect(fetchMock).toHaveBeenCalledTimes(1);
const body = String((fetchMock.mock.calls[0]![1] as RequestInit).body);
const parsed = JSON.parse(body);
expect(body).toBe(
JSON.stringify({
app: "paperclip",
schemaVersion: "1",
installId: readState().installId,
version: serverVersion,
events: [
{ name: "server.started", occurredAt: parsed.events[0].occurredAt, dimensions: { os_platform: process.platform } },
{
name: "agent.task_run",
occurredAt: parsed.events[1].occurredAt,
dimensions: { agent_id: "agent-1", state: "succeeded" },
},
],
batchId: parsed.batchId,
}),
);
expect(body).not.toContain("user-creator");
expect(body).not.toContain("attributed_via");
});
it("self-hosted: ignores posthog-js tracing headers", async () => { it("self-hosted: ignores posthog-js tracing headers", async () => {
expect(postHogRequestContextMiddlewareForEnv({})).toBeNull(); expect(postHogRequestContextMiddlewareForEnv({})).toBeNull();
expect(postHogRequestContextMiddlewareForEnv({ PAPERCLIP_POSTHOG_PROJECT_TOKEN: "phc_test" })).toBeNull(); expect(postHogRequestContextMiddlewareForEnv({ PAPERCLIP_POSTHOG_PROJECT_TOKEN: "phc_test" })).toBeNull();
@@ -222,12 +256,14 @@ describe("server telemetry sender selection", () => {
version: serverVersion, version: serverVersion,
stack_id: "stack-123", stack_id: "stack-123",
install_hash: installHash, install_hash: installHash,
attributed_via: "none",
environment: "staging", environment: "staging",
distinct_id: "workspace:stack-123", distinct_id: "workspace:stack-123",
$process_person_profile: false, $process_person_profile: false,
}); });
for (const message of posthog.captured) { for (const message of posthog.captured) {
expect(message.properties?.environment, message.event).toBe("staging"); expect(message.properties?.environment, message.event).toBe("staging");
if (message.event !== "$groupidentify") expect(message.properties?.attributed_via, message.event).toBe("none");
if (message.event !== "cloud.workspace_started") expect(message.properties).not.toHaveProperty("install_hash"); if (message.event !== "cloud.workspace_started") expect(message.properties).not.toHaveProperty("install_hash");
} }
expect(posthog.groups).toEqual([ expect(posthog.groups).toEqual([
@@ -432,4 +468,89 @@ describe("server telemetry sender selection", () => {
expect(routines[0]!.properties).toMatchObject({ company_id: "company-9", user_id: "user-1" }); expect(routines[0]!.properties).toMatchObject({ company_id: "company-9", user_id: "user-1" });
expect(routines[1]!.properties).toMatchObject({ company_id: "company-bg", user_id: "user-1" }); expect(routines[1]!.properties).toMatchObject({ company_id: "company-bg", user_id: "user-1" });
}); });
it("Cloud: resolves the person responsible for agent work in priority order", () => {
expect(resolveTelemetryPerson({
taskCreatedByUserId: "user-creator",
taskAssignedByUserId: "user-assigner",
runOwnerUserId: "user-owner",
})).toEqual({ userId: "user-creator", attributedVia: "task_creator" });
expect(resolveTelemetryPerson({ taskAssignedByUserId: "user-assigner", runOwnerUserId: "user-owner" }))
.toEqual({ userId: "user-assigner", attributedVia: "task_assigner" });
expect(resolveTelemetryPerson({ taskCreatedByUserId: " ", runOwnerUserId: "user-owner" }))
.toEqual({ userId: "user-owner", attributedVia: "run_owner" });
expect(resolveTelemetryPerson({ taskCreatedByUserId: "local-board", runOwnerUserId: null })).toBeNull();
expect(resolveTelemetryPerson({})).toBeNull();
});
it.each([
{ name: "task creator", input: { taskCreatedByUserId: "paperclip-id:creator", taskAssignedByUserId: "paperclip-id:assigner" }, distinctId: "paperclip-id:creator", via: "task_creator" },
{ name: "task assigner", input: { taskAssignedByUserId: "paperclip-id:assigner", runOwnerUserId: "paperclip-id:owner" }, distinctId: "paperclip-id:assigner", via: "task_assigner" },
{ name: "run owner", input: { runOwnerUserId: "paperclip-id:owner" }, distinctId: "paperclip-id:owner", via: "run_owner" },
])("Cloud: background agent work is attributed to the $name", async ({ input, distinctId, via }) => {
const posthog = fakePostHog();
const client = initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory })!;
await withTelemetryActor({ companyId: "company-bg", person: resolveTelemetryPerson(input) }, async () => {
await new Promise((resolve) => setTimeout(resolve, 1));
trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" });
});
await shutdownTelemetry();
const run = posthog.captured.find((message) => message.event === "agent.task_run")!;
expect(run.distinctId).toBe(distinctId);
expect(run.groups).toEqual({ workspace: "stack-123" });
expect(run.properties).toMatchObject({
company_id: "company-bg",
user_id: distinctId,
attributed_via: via,
distinct_id: distinctId,
});
expect(run.properties).not.toHaveProperty("$process_person_profile");
});
it("Cloud: background agent work with no person stays on the workspace", async () => {
const posthog = fakePostHog();
const client = initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory })!;
withTelemetryActor({ companyId: "company-bg", person: resolveTelemetryPerson({}) }, () =>
trackAgentTaskRun(client, { agentId: "agent-1", state: "succeeded" }));
await shutdownTelemetry();
const run = posthog.captured.find((message) => message.event === "agent.task_run")!;
expect(run.distinctId).toBe("workspace:stack-123");
expect(run.properties).toMatchObject({ attributed_via: "none", $process_person_profile: false });
expect(run.properties).not.toHaveProperty("user_id");
});
it("Cloud: the signed-in request user wins over a background person scope", async () => {
const posthog = fakePostHog();
initTelemetry({ enabled: true }, { env: CLOUD_ENV, postHogFactory: posthog.factory });
const app = express();
app.use(postHogRequestContextMiddlewareForEnv(CLOUD_ENV)!);
app.use((req, _res, next) => {
if (req.path === "/agent") req.actor = { type: "agent", agentId: "agent-1", companyId: "company-1", source: "agent_jwt" };
else req.actor = { type: "board", userId: "paperclip-id:request-user", source: "session" };
next();
});
const person = resolveTelemetryPerson({ taskCreatedByUserId: "paperclip-id:creator" });
app.post("/board", (_req, res) => {
withTelemetryActor({ companyId: "company-1", person }, () =>
trackAgentTaskRun(getTelemetryClient()!, { agentId: "agent-1", state: "cancelled" }));
res.json({});
});
app.post("/agent", (_req, res) => {
withTelemetryActor({ companyId: "company-1", person }, () =>
trackAgentTaskRun(getTelemetryClient()!, { agentId: "agent-1", state: "succeeded" }));
res.json({});
});
await request(app).post("/board").expect(200);
await request(app).post("/agent").expect(200);
await shutdownTelemetry();
const runs = posthog.captured.filter((message) => message.event === "agent.task_run");
const byState = Object.fromEntries(runs.map((message) => [message.properties?.state, message]));
expect(byState.cancelled!.distinctId).toBe("paperclip-id:request-user");
expect(byState.cancelled!.properties).toMatchObject({ attributed_via: "request" });
expect(byState.succeeded!.distinctId).toBe("paperclip-id:creator");
expect(byState.succeeded!.properties).toMatchObject({ attributed_via: "task_creator", company_id: "company-1" });
});
}); });
+7 -2
View File
@@ -208,7 +208,7 @@ import {
} from "@paperclipai/shared"; } from "@paperclipai/shared";
import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry"; import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry";
import { getTelemetryClient } from "../telemetry.js"; import { getTelemetryClient } from "../telemetry.js";
import { withTelemetryCompany } from "../telemetry-company-scope.js"; import { resolveTelemetryPerson, withTelemetryActor } from "../telemetry-company-scope.js";
import { isUniqueViolation } from "../db-errors.js"; import { isUniqueViolation } from "../db-errors.js";
import type { StorageService } from "../storage/types.js"; import type { StorageService } from "../storage/types.js";
import { validate, validateIssueMutationBody } from "../middleware/validate.js"; import { validate, validateIssueMutationBody } from "../middleware/validate.js";
@@ -16705,7 +16705,12 @@ export function issueRoutes(
typeof actorAgent.adapterConfig?.model === "string" typeof actorAgent.adapterConfig?.model === "string"
? actorAgent.adapterConfig.model ? actorAgent.adapterConfig.model
: undefined; : undefined;
withTelemetryCompany(issue.companyId, () => trackAgentTaskCompleted(tc, { const person = resolveTelemetryPerson({
taskCreatedByUserId: issue.createdByUserId,
taskAssignedByUserId: issue.responsibleUserId,
runOwnerUserId: req.actor.type === "agent" ? req.actor.onBehalfOfUserId : null,
});
withTelemetryActor({ companyId: issue.companyId, person }, () => trackAgentTaskCompleted(tc, {
agentRole: actorAgent.role, agentRole: actorAgent.role,
agentId: actorAgent.id, agentId: actorAgent.id,
adapterType: actorAgent.adapterType, adapterType: actorAgent.adapterType,
@@ -1,14 +1,17 @@
import { and, eq, sql } from "drizzle-orm"; import { and, eq, sql } from "drizzle-orm";
import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { agents, heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { trackAgentTaskRun } from "@paperclipai/shared/telemetry"; import { trackAgentTaskRun } from "@paperclipai/shared/telemetry";
import { parseObject } from "../adapters/utils.js"; import { parseObject } from "../adapters/utils.js";
import { logger } from "../middleware/logger.js"; import { logger } from "../middleware/logger.js";
import { getTelemetryClient } from "../telemetry.js"; import { getTelemetryClient } from "../telemetry.js";
import { withTelemetryCompany } from "../telemetry-company-scope.js"; import { resolveTelemetryPerson, withTelemetryActor } from "../telemetry-company-scope.js";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
type TelemetryRun = Pick<HeartbeatRun, type TelemetryRun = Pick<HeartbeatRun,
"id" | "companyId" | "agentId" | "status" | "startedAt" | "finishedAt" | "usageJson" | "contextSnapshot">; "id" | "companyId" | "agentId" | "status" | "startedAt" | "finishedAt" | "usageJson" | "contextSnapshot">
& Partial<Pick<HeartbeatRun, "responsibleUserId">>;
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
function readNonEmptyString(value: unknown): string | null { function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 return typeof value === "string" && value.trim().length > 0
@@ -60,13 +63,26 @@ export async function emitAgentTaskRun(db: Db, run: TelemetryRun): Promise<void>
const client = getTelemetryClient(); const client = getTelemetryClient();
if (!client) return; if (!client) return;
const taskId =
readNonEmptyString(parseObject(run.contextSnapshot).issueId) ??
undefined;
const taskRowId = taskId && UUID_RE.test(taskId) ? taskId : null;
const agent = await db const agent = await db
.select({ .select({
role: agents.role, role: agents.role,
adapterType: agents.adapterType, adapterType: agents.adapterType,
adapterConfig: agents.adapterConfig, adapterConfig: agents.adapterConfig,
taskCreatedByUserId: issues.createdByUserId,
taskResponsibleUserId: issues.responsibleUserId,
}) })
.from(agents) .from(agents)
.leftJoin(
issues,
taskRowId
? and(eq(issues.id, taskRowId), eq(issues.companyId, run.companyId))
: sql`false`,
)
.where(eq(agents.id, run.agentId)) .where(eq(agents.id, run.agentId))
.then((rows) => rows[0] ?? null); .then((rows) => rows[0] ?? null);
@@ -76,11 +92,13 @@ export async function emitAgentTaskRun(db: Db, run: TelemetryRun): Promise<void>
readNonEmptyString(parseObject(agent?.adapterConfig).model) ?? readNonEmptyString(parseObject(agent?.adapterConfig).model) ??
undefined; undefined;
const taskId = const person = resolveTelemetryPerson({
readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? taskCreatedByUserId: agent?.taskCreatedByUserId,
undefined; taskAssignedByUserId: agent?.taskResponsibleUserId,
runOwnerUserId: run.responsibleUserId,
});
withTelemetryCompany(run.companyId, () => trackAgentTaskRun(client, { withTelemetryActor({ companyId: run.companyId, person }, () => trackAgentTaskRun(client, {
agentId: run.agentId, agentId: run.agentId,
state: run.status, state: run.status,
...(agent ...(agent
@@ -141,6 +159,7 @@ export async function emitAgentTaskRunById(
startedAt: heartbeatRuns.startedAt, startedAt: heartbeatRuns.startedAt,
finishedAt: heartbeatRuns.finishedAt, finishedAt: heartbeatRuns.finishedAt,
usageJson: heartbeatRuns.usageJson, usageJson: heartbeatRuns.usageJson,
responsibleUserId: heartbeatRuns.responsibleUserId,
contextSnapshot: sql<Record<string, unknown>>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId')`, contextSnapshot: sql<Record<string, unknown>>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId')`,
}) })
.from(heartbeatRuns) .from(heartbeatRuns)
@@ -5,7 +5,7 @@ import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm";
import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry"; import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry";
import { getTelemetryClient } from "../../telemetry.js"; import { getTelemetryClient } from "../../telemetry.js";
import { withTelemetryCompany } from "../../telemetry-company-scope.js"; import { resolveTelemetryPerson, withTelemetryActor } from "../../telemetry-company-scope.js";
import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
import { publishLiveEvent } from "../live-events.js"; import { publishLiveEvent } from "../live-events.js";
import { finishedIdleRun, finishedIdleWake, finishedFailedIdleLease } from "../idle-work-proof.js"; import { finishedIdleRun, finishedIdleWake, finishedFailedIdleLease } from "../idle-work-proof.js";
@@ -117,7 +117,8 @@ export function createHeartbeatAgentStatus(db: Db) {
if (!result?.updated) return; if (!result?.updated) return;
if (result.isFirstHeartbeat) { if (result.isFirstHeartbeat) {
const client = getTelemetryClient(); const client = getTelemetryClient();
if (client) withTelemetryCompany(result.updated.companyId, () => trackAgentFirstHeartbeat(client, { agentRole: result.updated.role, agentId: result.updated.id })); const person = resolveTelemetryPerson({ runOwnerUserId: result.updated.createdByUserId });
if (client) withTelemetryActor({ companyId: result.updated.companyId, person }, () => trackAgentFirstHeartbeat(client, { agentRole: result.updated.role, agentId: result.updated.id }));
} }
publishStatus(result.updated, outcome); publishStatus(result.updated, outcome);
} }
@@ -179,6 +179,7 @@ export function buildPostHogDirectEvent(
stack_id: config.stackId, stack_id: config.stackId,
...(context?.companyId ? { company_id: context.companyId } : {}), ...(context?.companyId ? { company_id: context.companyId } : {}),
...(context?.userId ? { user_id: context.userId } : {}), ...(context?.userId ? { user_id: context.userId } : {}),
attributed_via: context?.userId ? (context.attributedVia ?? "request") : "none",
...(config.environment ? { environment: config.environment } : {}), ...(config.environment ? { environment: config.environment } : {}),
...(context?.sessionId ? { $session_id: context.sessionId } : {}), ...(context?.sessionId ? { $session_id: context.sessionId } : {}),
...(context?.windowId ? { $window_id: context.windowId } : {}), ...(context?.windowId ? { $window_id: context.windowId } : {}),
@@ -224,6 +225,7 @@ export function buildWorkspaceStartedEvent(
...(meta.version ? { version: meta.version } : {}), ...(meta.version ? { version: meta.version } : {}),
stack_id: config.stackId, stack_id: config.stackId,
install_hash: input.installHash, install_hash: input.installHash,
attributed_via: "none",
...(config.environment ? { environment: config.environment } : {}), ...(config.environment ? { environment: config.environment } : {}),
$groups: { [POSTHOG_WORKSPACE_GROUP_TYPE]: config.stackId }, $groups: { [POSTHOG_WORKSPACE_GROUP_TYPE]: config.stackId },
distinct_id: distinctId, distinct_id: distinctId,
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from "node:async_hooks"; import { AsyncLocalStorage } from "node:async_hooks";
import type { Request, RequestHandler } from "express"; import type { Request, RequestHandler } from "express";
import { isPostHogDirectDeliveryEnabled, resolveCloudPostHogConfig } from "../cloud-posthog-config.js"; import { isPostHogDirectDeliveryEnabled, resolveCloudPostHogConfig } from "../cloud-posthog-config.js";
import { scopedTelemetryCompanyId } from "../../telemetry-company-scope.js"; import { scopedTelemetryCompanyId, scopedTelemetryPerson } from "../../telemetry-company-scope.js";
/** /**
* Request-scoped analytics context for Paperclip Cloud direct delivery. * Request-scoped analytics context for Paperclip Cloud direct delivery.
@@ -21,8 +21,11 @@ type PostHogRequestStore = {
active: boolean; active: boolean;
}; };
export type PostHogAttribution = "request" | "task_creator" | "task_assigner" | "run_owner" | "none";
export type PostHogEventContext = { export type PostHogEventContext = {
userId?: string; userId?: string;
attributedVia?: PostHogAttribution;
companyId?: string; companyId?: string;
sessionId?: string; sessionId?: string;
windowId?: string; windowId?: string;
@@ -103,11 +106,21 @@ export function capturePostHogEventContext(): PostHogEventContext | undefined {
if (store?.active) { if (store?.active) {
const userId = actingUserId(store.req); const userId = actingUserId(store.req);
const companyId = knownCompanyId(store.req); const companyId = knownCompanyId(store.req);
if (userId) context.userId = userId; if (userId) {
context.userId = userId;
context.attributedVia = "request";
}
if (companyId) context.companyId = companyId; if (companyId) context.companyId = companyId;
if (store.sessionId) context.sessionId = store.sessionId; if (store.sessionId) context.sessionId = store.sessionId;
if (store.windowId) context.windowId = store.windowId; if (store.windowId) context.windowId = store.windowId;
} }
if (!context.userId) {
const person = scopedTelemetryPerson();
if (person) {
context.userId = person.userId;
context.attributedVia = person.attributedVia;
}
}
if (!context.companyId) { if (!context.companyId) {
const scopedCompanyId = scopedTelemetryCompanyId(); const scopedCompanyId = scopedTelemetryCompanyId();
if (scopedCompanyId) context.companyId = scopedCompanyId; if (scopedCompanyId) context.companyId = scopedCompanyId;
+49 -3
View File
@@ -1,11 +1,57 @@
import { AsyncLocalStorage } from "node:async_hooks"; import { AsyncLocalStorage } from "node:async_hooks";
const storage = new AsyncLocalStorage<string>(); export type TelemetryPersonAttribution = "task_creator" | "task_assigner" | "run_owner";
export type TelemetryPerson = {
userId: string;
attributedVia: TelemetryPersonAttribution;
};
type TelemetryScope = {
companyId?: string;
person?: TelemetryPerson;
};
const storage = new AsyncLocalStorage<TelemetryScope>();
const NON_PERSON_USER_IDS = new Set(["local-board"]);
export function withTelemetryCompany<T>(companyId: string | null | undefined, fn: () => T): T { export function withTelemetryCompany<T>(companyId: string | null | undefined, fn: () => T): T {
return companyId ? storage.run(companyId, fn) : fn(); return withTelemetryActor({ companyId }, fn);
}
export function withTelemetryActor<T>(
scope: { companyId?: string | null; person?: TelemetryPerson | null },
fn: () => T,
): T {
const next: TelemetryScope = {};
if (scope.companyId) next.companyId = scope.companyId;
if (scope.person) next.person = scope.person;
return next.companyId || next.person ? storage.run(next, fn) : fn();
} }
export function scopedTelemetryCompanyId(): string | undefined { export function scopedTelemetryCompanyId(): string | undefined {
return storage.getStore(); return storage.getStore()?.companyId;
}
export function scopedTelemetryPerson(): TelemetryPerson | undefined {
return storage.getStore()?.person;
}
function personUserId(value: string | null | undefined): string | null {
const trimmed = value?.trim();
return trimmed && !NON_PERSON_USER_IDS.has(trimmed) ? trimmed : null;
}
export function resolveTelemetryPerson(input: {
taskCreatedByUserId?: string | null;
taskAssignedByUserId?: string | null;
runOwnerUserId?: string | null;
}): TelemetryPerson | null {
const creator = personUserId(input.taskCreatedByUserId);
if (creator) return { userId: creator, attributedVia: "task_creator" };
const assigner = personUserId(input.taskAssignedByUserId);
if (assigner) return { userId: assigner, attributedVia: "task_assigner" };
const runOwner = personUserId(input.runOwnerUserId);
if (runOwner) return { userId: runOwner, attributedVia: "run_owner" };
return null;
} }
@@ -370,7 +370,7 @@
"packages/shared/src/standing-checks.ts": "7ab71e4b6984e41e1c1dfd1bddf0116041d5abf1", "packages/shared/src/standing-checks.ts": "7ab71e4b6984e41e1c1dfd1bddf0116041d5abf1",
"packages/shared/src/subscriptions.ts": "ef70e63395ca967bbc30776b71560e96eb354a75", "packages/shared/src/subscriptions.ts": "ef70e63395ca967bbc30776b71560e96eb354a75",
"packages/shared/src/summary-slot.test.ts": "5f8e222dea425f0d5bff0adb5c683087bd7cf7fb", "packages/shared/src/summary-slot.test.ts": "5f8e222dea425f0d5bff0adb5c683087bd7cf7fb",
"packages/shared/src/telemetry/README.md": "eaa75efee24352f97ddc7d72b2e99be8b39a241b", "packages/shared/src/telemetry/README.md": "99c1d1eddf1e5133bed005fcdef6dbfe12d83460",
"packages/shared/src/telemetry/client-types.test.ts": "3b630edda4972b929ba4decd7457f226eb9cfc5b", "packages/shared/src/telemetry/client-types.test.ts": "3b630edda4972b929ba4decd7457f226eb9cfc5b",
"packages/shared/src/telemetry/client.test.ts": "5a95f2305a393c5b2e7750b10e0ed65da0f30cf8", "packages/shared/src/telemetry/client.test.ts": "5a95f2305a393c5b2e7750b10e0ed65da0f30cf8",
"packages/shared/src/telemetry/client.ts": "d6534ce8aaa6f69fb2dd03e12487db024e250a96", "packages/shared/src/telemetry/client.ts": "d6534ce8aaa6f69fb2dd03e12487db024e250a96",
@@ -208,6 +208,11 @@ vi.mock("../api/approvals", () => ({ approvalsApi: mockApprovalsApi }));
vi.mock("../api/secrets", () => ({ secretsApi: mockSecretsApi })); vi.mock("../api/secrets", () => ({ secretsApi: mockSecretsApi }));
vi.mock("../api/issues", () => ({ issuesApi: mockIssuesApi })); vi.mock("../api/issues", () => ({ issuesApi: mockIssuesApi }));
vi.mock("../api/agentChats", () => ({ agentChatsApi: mockAgentChatsApi })); vi.mock("../api/agentChats", () => ({ agentChatsApi: mockAgentChatsApi }));
const mockCaptureProductEvent = vi.hoisted(() => vi.fn());
vi.mock("../lib/analytics/product-analytics", async (importOriginal) => ({
...(await importOriginal<typeof import("../lib/analytics/product-analytics")>()),
captureProductAnalyticsEvent: mockCaptureProductEvent,
}));
vi.mock("../api/projects", () => ({ projectsApi: mockProjectsApi })); vi.mock("../api/projects", () => ({ projectsApi: mockProjectsApi }));
vi.mock("../api/environments", () => ({ environmentsApi: mockEnvironmentsApi })); vi.mock("../api/environments", () => ({ environmentsApi: mockEnvironmentsApi }));
vi.mock("../api/instanceSettings", () => ({ instanceSettingsApi: mockInstanceSettingsApi })); vi.mock("../api/instanceSettings", () => ({ instanceSettingsApi: mockInstanceSettingsApi }));
@@ -1465,6 +1470,26 @@ describe("OnboardingWizard restore-gate (stale localStorage across accounts)", (
} finally { await act(async () => root.unmount()); } } finally { await act(async () => root.unmount()); }
}); });
it("keeps onboarding analytics on the new company when the old launch settles late", async () => {
let settleLaunch!: (value: { id: string }) => void;
mockAgentChatsApi.startOnboarding.mockImplementationOnce(() => new Promise((resolve) => { settleLaunch = resolve; }));
const { root, queryClient } = await openSavedReview("/SC/onboarding");
try {
await act(async () => [...document.querySelectorAll("button")].find(button => button.textContent?.trim() === "Get started")!.click());
mockRouterState.pathname = "/OTHER/onboarding";
await renderWizardView(root, queryClient);
for (let tick = 0; tick < 8; tick++) await flushReact();
await act(async () => settleLaunch({ id: "onboarding-chat" }));
for (let tick = 0; tick < 4; tick++) await flushReact();
expect(mockNavigate).not.toHaveBeenCalled();
mockCaptureProductEvent.mockClear();
mockRouterState.pathname = "/OTHER/dashboard";
await renderWizardView(root, queryClient);
for (let tick = 0; tick < 4; tick++) await flushReact();
expect(mockCaptureProductEvent).toHaveBeenCalledWith("onboarding_dismissed", { step: "create_agent" });
} finally { await act(async () => root.unmount()); }
});
it("keeps restored review progress when initial effects replay in StrictMode", async () => { it("keeps restored review progress when initial effects replay in StrictMode", async () => {
const { root, draft } = await openSavedReview("/SC/onboarding", "auto", { strict: true }); const { root, draft } = await openSavedReview("/SC/onboarding", "auto", { strict: true });
try { try {
+14
View File
@@ -140,6 +140,7 @@ import { OnboardingHeading } from "./onboarding/OnboardingPrimitives";
import { DEFAULT_AGENT_ROLE } from "../lib/onboarding-agent-role"; import { DEFAULT_AGENT_ROLE } from "../lib/onboarding-agent-role";
import { capsuleHeroMotion, capsuleRoomEnter, capsuleRoomExit, heroRoomArrival, heroRoomMotion, ledeMotion, stepContentMotion, titleSwapMotion } from "./onboarding/onboarding-motion"; import { capsuleHeroMotion, capsuleRoomEnter, capsuleRoomExit, heroRoomArrival, heroRoomMotion, ledeMotion, stepContentMotion, titleSwapMotion } from "./onboarding/onboarding-motion";
import { Badge } from "@/components/ui/badge"; import { Badge } from "@/components/ui/badge";
import { captureOnboardingStepCompleted, useOnboardingAnalytics } from "@/lib/analytics/onboarding-analytics";
import { import {
ArrowLeft, ArrowLeft,
ArrowRight, ArrowRight,
@@ -677,6 +678,11 @@ function OnboardingWizardInner({
(savedKeys.subscriptions.length > 0 || (adapterType === "claude_local" && savedKeys.storedLogin.data)) (savedKeys.subscriptions.length > 0 || (adapterType === "claude_local" && savedKeys.storedLogin.data))
? "subscription" : savedKeys.options.length || adapterType === "opencode_local" ? "api" : "subscription" ? "subscription" : savedKeys.options.length || adapterType === "opencode_local" ? "api" : "subscription"
); );
const onboardingAnalytics = useOnboardingAnalytics({
open: effectiveOnboardingOpen,
step,
connectOutcome: credentialMode === "api" ? "api_key" : "subscription",
});
const [createdCompanyPrefix, setCreatedCompanyPrefix] = useState< const [createdCompanyPrefix, setCreatedCompanyPrefix] = useState<
string | null string | null
>((saved?.createdCompanyPrefix as string) ?? null); >((saved?.createdCompanyPrefix as string) ?? null);
@@ -830,6 +836,9 @@ function OnboardingWizardInner({
setStep(initialStepRef.current); setStep(initialStepRef.current);
setEntryStep(initialStepRef.current); setEntryStep(initialStepRef.current);
} }
if (!previousRequest?.open) {
onboardingAnalytics.opened(initialStepRef.current && !resumesRouteDraft ? initialStepRef.current : step, step);
}
if (routeCompanyId) { if (routeCompanyId) {
// Claim ownership only when the route *introduces* a company. A route // Claim ownership only when the route *introduces* a company. A route
// that merely names the one already in hand - the wizard created it, // that merely names the one already in hand - the wizard created it,
@@ -1831,6 +1840,7 @@ function OnboardingWizardInner({
} }
function handleClose() { function handleClose() {
onboardingAnalytics.dismissed(step);
reset(); reset();
closeOnboarding(); closeOnboarding();
// On the /onboarding route the wizard is also kept open by the route // On the /onboarding route the wizard is also kept open by the route
@@ -1884,9 +1894,12 @@ function OnboardingWizardInner({
} }
setLoading(true); setLoading(true);
setError(null); setError(null);
captureOnboardingStepCompleted("review");
try { try {
await agentChatsApi.startOnboarding(createdCompanyId, createdAgentId); await agentChatsApi.startOnboarding(createdCompanyId, createdAgentId);
captureOnboardingStepCompleted("launch", "succeeded");
if (!stillTheSameCompany(createdCompanyId)) return; if (!stillTheSameCompany(createdCompanyId)) return;
onboardingAnalytics.finished();
const prefix = createdCompanyPrefix; const prefix = createdCompanyPrefix;
const agentId = createdAgentId; const agentId = createdAgentId;
// Route sync preserves this explicit destination over page memory. // Route sync preserves this explicit destination over page memory.
@@ -1895,6 +1908,7 @@ function OnboardingWizardInner({
closeOnboarding(); closeOnboarding();
navigate(`${prefix ? `/${prefix}` : ""}/chats/${agentId}`); navigate(`${prefix ? `/${prefix}` : ""}/chats/${agentId}`);
} catch (err) { } catch (err) {
captureOnboardingStepCompleted("launch", "failed");
setError(err instanceof Error ? err.message : "Failed to open your chat. Please try again."); setError(err instanceof Error ? err.message : "Failed to open your chat. Please try again.");
} finally { } finally {
setLoading(false); setLoading(false);
@@ -0,0 +1,260 @@
// @vitest-environment jsdom
import { act, createElement } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { BrowserAnalyticsConfig } from "@paperclipai/shared";
import {
captureProductAnalyticsEvent,
resetProductAnalyticsForTest,
setProductAnalyticsUser,
syncProductAnalytics,
type ProductAnalyticsClient,
} from "./product-analytics";
import { captureOnboardingStepCompleted, useOnboardingAnalytics } from "./onboarding-analytics";
(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
const unmounts: Array<() => void> = [];
function renderHook<R, P>(callback: (props: P) => R, options: { initialProps: P }) {
const host = document.createElement("div");
document.body.append(host);
const root = createRoot(host);
let props = options.initialProps;
const result = {} as { current: R };
function Harness() {
result.current = callback(props);
return null;
}
function rerender(next: P = props) {
props = next;
act(() => root.render(createElement(Harness)));
}
unmounts.push(() => {
act(() => root.unmount());
host.remove();
});
rerender();
return { result, rerender };
}
const config: BrowserAnalyticsConfig = {
token: "phc_test",
apiHost: "/_ph",
uiHost: "https://us.posthog.com",
environment: "staging",
stackId: "stack-1",
replayEnabled: false,
};
function mockClient() {
return {
init: vi.fn(),
register: vi.fn(),
identify: vi.fn(),
group: vi.fn(),
reset: vi.fn(),
capture: vi.fn(),
opt_out_capturing: vi.fn(),
opt_in_capturing: vi.fn(),
};
}
async function flush() {
await Promise.resolve();
await Promise.resolve();
}
let client: ReturnType<typeof mockClient>;
let scheduled: (() => void) | null;
const load = () => Promise.resolve(client as unknown as ProductAnalyticsClient);
const schedule = (start: () => void) => {
scheduled = start;
return () => { scheduled = null; };
};
async function startAnalytics() {
syncProductAnalytics(config, "granted", { load, schedule, immediate: true });
await flush();
}
function captured() {
return client.capture.mock.calls.map(([event, properties]) => [event, properties]);
}
beforeEach(() => {
client = mockClient();
scheduled = null;
});
afterEach(() => {
for (const unmount of unmounts.splice(0)) unmount();
resetProductAnalyticsForTest();
});
describe("browser product events", () => {
it("captures nothing when browser analytics is off", async () => {
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
await flush();
expect(client.capture).not.toHaveBeenCalled();
});
it.each(["denied", "ask"] as const)("captures nothing when consent is %s", async (consent) => {
syncProductAnalytics(config, consent, { load, schedule, immediate: true });
await flush();
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
await flush();
expect(client.init).not.toHaveBeenCalled();
expect(client.capture).not.toHaveBeenCalled();
});
it("holds events from before posthog-js loads and sends them with their own time", async () => {
syncProductAnalytics(config, "granted", { load, schedule });
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
expect(client.capture).not.toHaveBeenCalled();
scheduled!();
await flush();
expect(client.capture).toHaveBeenCalledTimes(1);
expect(client.capture).toHaveBeenCalledWith(
"onboarding_opened",
{ step: "create_agent" },
{ timestamp: expect.any(Date) },
);
});
it("drops held events when consent is withdrawn before posthog-js loads", async () => {
syncProductAnalytics(config, "granted", { load, schedule });
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
scheduled!();
syncProductAnalytics(config, "denied", { load, schedule });
await flush();
captureProductAnalyticsEvent("onboarding_dismissed", { step: "create_agent" });
expect(client.capture).not.toHaveBeenCalled();
});
it("drops held events when the signed-in person changes before posthog-js loads", async () => {
setProductAnalyticsUser({ id: "user-1", email: null, name: null });
syncProductAnalytics(config, "granted", { load, schedule });
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
setProductAnalyticsUser(null);
setProductAnalyticsUser({ id: "user-2", email: null, name: null });
captureProductAnalyticsEvent("onboarding_opened", { step: "connect_model" });
scheduled!();
await flush();
expect(captured()).toEqual([["onboarding_opened", { step: "connect_model" }]]);
expect(client.identify).toHaveBeenCalledWith("user-2", {});
});
it("keeps held events when the same person is set again before posthog-js loads", async () => {
setProductAnalyticsUser({ id: "user-1", email: null, name: null });
syncProductAnalytics(config, "granted", { load, schedule });
captureProductAnalyticsEvent("onboarding_opened", { step: "create_agent" });
setProductAnalyticsUser({ id: "user-1", email: "jane@example.com", name: null });
scheduled!();
await flush();
expect(captured()).toEqual([["onboarding_opened", { step: "create_agent" }]]);
});
it("stops capturing after consent is withdrawn", async () => {
await startAnalytics();
syncProductAnalytics(config, "denied", { load, schedule });
captureProductAnalyticsEvent("onboarding_opened", {});
expect(client.capture).not.toHaveBeenCalled();
});
});
describe("onboarding funnel events", () => {
function renderWizard(initial: { open: boolean; step: number }) {
return renderHook(
(props: { open: boolean; step: number; connectOutcome: "subscription" | "api_key" }) => useOnboardingAnalytics(props),
{ initialProps: { ...initial, connectOutcome: "subscription" as "subscription" | "api_key" } },
);
}
it("captures each transition once through the wizard", async () => {
await startAnalytics();
const wizard = renderWizard({ open: false, step: 1 });
wizard.rerender({ open: true, step: 1, connectOutcome: "subscription" });
act(() => wizard.result.current.opened(3, 1));
wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" });
wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" });
wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" });
wizard.rerender({ open: true, step: 4, connectOutcome: "api_key" });
wizard.rerender({ open: true, step: 5, connectOutcome: "api_key" });
captureOnboardingStepCompleted("review");
captureOnboardingStepCompleted("launch", "succeeded");
act(() => wizard.result.current.finished());
wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" });
expect(captured()).toEqual([
["onboarding_opened", { step: "create_agent" }],
["onboarding_step_viewed", { step: "create_agent" }],
["onboarding_step_completed", { step: "create_agent" }],
["onboarding_step_viewed", { step: "connect_model" }],
["onboarding_step_completed", { step: "connect_model", outcome: "api_key" }],
["onboarding_step_viewed", { step: "review" }],
["onboarding_step_completed", { step: "review" }],
["onboarding_step_completed", { step: "launch", outcome: "succeeded" }],
]);
});
it("does not count a stale step from the opening render as a view", async () => {
await startAnalytics();
const wizard = renderWizard({ open: true, step: 5 });
act(() => wizard.result.current.opened(3, 5));
wizard.rerender({ open: true, step: 5, connectOutcome: "subscription" });
wizard.rerender({ open: true, step: 3, connectOutcome: "subscription" });
expect(captured()).toEqual([
["onboarding_opened", { step: "create_agent" }],
["onboarding_step_viewed", { step: "create_agent" }],
]);
});
it("captures a dismissal once and a reopen as a new session", async () => {
await startAnalytics();
const wizard = renderWizard({ open: true, step: 4 });
act(() => wizard.result.current.opened(4, 4));
act(() => wizard.result.current.opened(4, 4));
act(() => wizard.result.current.dismissed(4));
act(() => wizard.result.current.dismissed(4));
wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" });
wizard.rerender({ open: true, step: 1, connectOutcome: "subscription" });
act(() => wizard.result.current.opened(3, 1));
expect(captured()).toEqual([
["onboarding_opened", { step: "connect_model" }],
["onboarding_step_viewed", { step: "connect_model" }],
["onboarding_dismissed", { step: "connect_model" }],
["onboarding_opened", { step: "create_agent" }],
["onboarding_step_viewed", { step: "create_agent" }],
]);
});
it("captures a dismissal when the wizard closes before launch", async () => {
await startAnalytics();
const wizard = renderWizard({ open: true, step: 3 });
act(() => wizard.result.current.opened(3, 3));
wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" });
wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" });
wizard.rerender({ open: false, step: 1, connectOutcome: "subscription" });
expect(captured()).toEqual([
["onboarding_opened", { step: "create_agent" }],
["onboarding_step_viewed", { step: "create_agent" }],
["onboarding_step_completed", { step: "create_agent" }],
["onboarding_step_viewed", { step: "connect_model" }],
["onboarding_dismissed", { step: "connect_model" }],
]);
});
it("captures nothing in the wizard when consent is not granted", async () => {
syncProductAnalytics(config, "ask", { load, schedule, immediate: true });
const wizard = renderWizard({ open: true, step: 3 });
act(() => wizard.result.current.opened(3, 3));
wizard.rerender({ open: true, step: 4, connectOutcome: "subscription" });
act(() => wizard.result.current.dismissed(4));
expect(client.capture).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,92 @@
import { useCallback, useEffect, useRef } from "react";
import { captureProductAnalyticsEvent } from "./product-analytics";
export const ONBOARDING_ANALYTICS_EVENTS = {
opened: "onboarding_opened",
stepViewed: "onboarding_step_viewed",
stepCompleted: "onboarding_step_completed",
dismissed: "onboarding_dismissed",
} as const;
export type OnboardingAnalyticsStep = "create_agent" | "connect_model" | "review" | "launch";
export type OnboardingAnalyticsOutcome = "subscription" | "api_key" | "succeeded" | "failed";
const WIZARD_STEPS: Record<number, OnboardingAnalyticsStep> = {
3: "create_agent",
4: "connect_model",
5: "review",
};
export function onboardingAnalyticsStepFor(wizardStep: number): OnboardingAnalyticsStep | null {
return WIZARD_STEPS[wizardStep] ?? null;
}
function stepProperties(step: OnboardingAnalyticsStep | null, outcome?: OnboardingAnalyticsOutcome) {
const properties: Record<string, string> = {};
if (step) properties.step = step;
if (outcome) properties.outcome = outcome;
return properties;
}
export function captureOnboardingStepCompleted(step: OnboardingAnalyticsStep, outcome?: OnboardingAnalyticsOutcome) {
captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepCompleted, stepProperties(step, outcome));
}
export function useOnboardingAnalytics(input: {
open: boolean;
step: number;
connectOutcome: OnboardingAnalyticsOutcome;
}) {
const session = useRef<{ pendingStep: number | null; viewedStep: number } | null>(null);
const connectOutcome = useRef(input.connectOutcome);
connectOutcome.current = input.connectOutcome;
const opened = useCallback((entryStep: number, renderedStep: number) => {
if (session.current) return;
session.current = { pendingStep: entryStep === renderedStep ? null : entryStep, viewedStep: entryStep };
const step = onboardingAnalyticsStepFor(entryStep);
captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.opened, stepProperties(step));
if (step) captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepViewed, stepProperties(step));
}, []);
const dismissed = useCallback((step: number) => {
if (!session.current) return;
session.current = null;
captureProductAnalyticsEvent(
ONBOARDING_ANALYTICS_EVENTS.dismissed,
stepProperties(onboardingAnalyticsStepFor(step)),
);
}, []);
const finished = useCallback(() => {
session.current = null;
}, []);
const { open, step } = input;
useEffect(() => {
const current = session.current;
if (!open) {
if (current) dismissed(current.viewedStep);
return;
}
if (!current) return;
if (current.pendingStep !== null) {
if (step !== current.pendingStep) return;
current.pendingStep = null;
}
const previous = current.viewedStep;
if (previous === step) return;
current.viewedStep = step;
const previousStep = onboardingAnalyticsStepFor(previous);
if (previousStep && step === previous + 1) {
captureOnboardingStepCompleted(
previousStep,
previousStep === "connect_model" ? connectOutcome.current : undefined,
);
}
const viewed = onboardingAnalyticsStepFor(step);
if (viewed) captureProductAnalyticsEvent(ONBOARDING_ANALYTICS_EVENTS.stepViewed, stepProperties(viewed));
}, [open, step]);
return { opened, dismissed, finished };
}
+31
View File
@@ -37,6 +37,7 @@ export interface ProductAnalyticsClient {
identify(distinctId: string, personProperties?: Record<string, unknown>): void; identify(distinctId: string, personProperties?: Record<string, unknown>): void;
group(groupType: string, groupKey: string): void; group(groupType: string, groupKey: string): void;
reset(): void; reset(): void;
capture?(event: string, properties?: Record<string, unknown>, options?: { timestamp?: Date }): unknown;
opt_out_capturing?(): void; opt_out_capturing?(): void;
opt_in_capturing?(): void; opt_in_capturing?(): void;
has_opted_out_capturing?(): boolean; has_opted_out_capturing?(): boolean;
@@ -210,8 +211,17 @@ type State = {
pendingClient: ProductAnalyticsClient | null; pendingClient: ProductAnalyticsClient | null;
loading: boolean; loading: boolean;
optedOut: boolean; optedOut: boolean;
queued: QueuedProductEvent[];
}; };
type QueuedProductEvent = {
event: string;
properties: Record<string, string>;
timestamp: Date;
};
const MAX_QUEUED_PRODUCT_EVENTS = 50;
let state: State | null = null; let state: State | null = null;
let wantedUser: ProductAnalyticsUser | null = null; let wantedUser: ProductAnalyticsUser | null = null;
let identifiedUserId: string | null = null; let identifiedUserId: string | null = null;
@@ -236,6 +246,21 @@ function applyIdentity(client: ProductAnalyticsClient, config: BrowserAnalyticsC
} }
} }
function sendProductEvent(client: ProductAnalyticsClient, queued: QueuedProductEvent) {
client.capture?.(queued.event, queued.properties, { timestamp: queued.timestamp });
}
export function captureProductAnalyticsEvent(event: string, properties: Record<string, string> = {}): void {
const current = state;
if (!current || current.optedOut) return;
const queued = { event, properties: { ...properties }, timestamp: new Date() };
if (current.client) {
sendProductEvent(current.client, queued);
return;
}
if (current.queued.length < MAX_QUEUED_PRODUCT_EVENTS) current.queued.push(queued);
}
function optOut(client: ProductAnalyticsClient) { function optOut(client: ProductAnalyticsClient) {
client.opt_out_capturing?.(); client.opt_out_capturing?.();
client.stopSessionRecording?.(); client.stopSessionRecording?.();
@@ -253,6 +278,9 @@ function initialize(current: State, client: ProductAnalyticsClient) {
client.register(superProperties(config)); client.register(superProperties(config));
current.client = client; current.client = client;
applyIdentity(client, config); applyIdentity(client, config);
const queued = current.queued;
current.queued = [];
for (const event of queued) sendProductEvent(client, event);
} }
/** /**
@@ -283,6 +311,7 @@ export function startProductAnalytics(
cancelSchedule: null, cancelSchedule: null,
loading: false, loading: false,
optedOut: false, optedOut: false,
queued: [],
}; };
state = current; state = current;
const load = options.load ?? loadPostHog; const load = options.load ?? loadPostHog;
@@ -309,6 +338,7 @@ export function startProductAnalytics(
/** Set (or clear, on sign-out) the signed-in user. Safe to call before posthog-js loads. */ /** Set (or clear, on sign-out) the signed-in user. Safe to call before posthog-js loads. */
export function setProductAnalyticsUser(user: ProductAnalyticsUser | null): void { export function setProductAnalyticsUser(user: ProductAnalyticsUser | null): void {
if (wantedUser && wantedUser.id !== user?.id && state) state.queued = [];
wantedUser = user; wantedUser = user;
if (state?.client) applyIdentity(state.client, state.config); if (state?.client) applyIdentity(state.client, state.config);
} }
@@ -341,6 +371,7 @@ function stopProductAnalytics(): void {
return; return;
} }
state.optedOut = true; state.optedOut = true;
state.queued = [];
if (state.client) optOut(state.client); if (state.client) optOut(state.client);
} }