diff --git a/.dockerignore b/.dockerignore index cb547de2a60..2b257d8eb55 100644 --- a/.dockerignore +++ b/.dockerignore @@ -81,3 +81,9 @@ packages/grok-acp/bin !packages/paperclip-runner/test-fixtures/pi-acp/profile-v24-claude-historical-identity.json !packages/paperclip-runner/test-fixtures/pi-acp/profile-v27-identity.json + +!packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json + +!packages/paperclip-runner/test/fixtures/copilot-profile-v23-identity.json + +!packages/paperclip-runner/test/fixtures/copilot-profile-v24-identity.json diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index fb874f7f708..376e58a6484 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -68,6 +68,12 @@ V1 implementation extends this baseline into a company-centric, governance-aware ## 5.1 In Scope +Company administrators may opt in to native Codex and Claude Code logs/traces +through [company agent observability](agent-observability.md), using a single +company destination and encrypted credentials. This is separate from first-party +Telemetry and infrastructure tracing. + + - Company lifecycle (create/list/get/update/archive) - Goal hierarchy linked to company mission - Agent lifecycle with org structure and adapter configuration diff --git a/doc/agent-observability.md b/doc/agent-observability.md new file mode 100644 index 00000000000..2932330f2dc --- /dev/null +++ b/doc/agent-observability.md @@ -0,0 +1,105 @@ +# Company agent observability + +Company Settings → General → Agent observability configures one company-owned +OTLP destination for Paperclip-managed Codex and Claude Code executions. Export +is disabled by default. Choose Braintrust (US) or a custom HTTP(S) OTLP base URL; +use your own collector when you need fan-out. Braintrust EU and self-hosted +installations can use Custom OTLP with their required routing/authentication +headers. The receiver must be reachable from every execution environment; +`localhost` refers to the execution host, including a remote sandbox. + +This is separate from Paperclip's first-party Telemetry and its content-free +[infrastructure tracing](observability.md). Configuring it does not change either data path. + +## Capture and supported runtimes + +The qualified versions are Codex 0.160.0 and Claude Code through ACP bridge +0.73.0 with Agent SDK 0.3.286. Both logs and traces use HTTP/protobuf. Prompts, +assistant responses and tool content default on. Additional tool details are a +Claude Code control; Codex has no equivalent switch. Codex requests up to 64 KiB +of tool output (or zero when disabled); native event availability, truncation +and response caps still apply. Claude Code applies its own native limits. +Metrics, raw API bodies and Codex Guardian assessments are disabled. + +The native runner and legacy local adapters receive transient launch material. +Native Codex writes a private managed config.toml; legacy exec uses a unique +invocation profile, preserving a selected source profile. Its profile injection +requires the qualified executable version. Codex ACPX writes its isolated home. +The legacy ACPX adapter keeps export configuration in a private home per agent +and settings compatibility key, and shares conversation files with its stable managed home; +export changes do not change its logical conversation fingerprint. +Codex's bundled Rust OTLP exporter percent-decodes header values, so launch +material encodes them once to preserve literal percent sequences. Credentials +remain literal values in the company secret; environment interpolation is not used. +A caller-owned/preconfigured Codex home cannot be automatically configured in +this version: its files remain untouched and execution emits a diagnostic. +Other harnesses, personal Codex desktop sessions and hosted claude_managed are +outside this feature. + +## Changes and process lifetime + +Paperclip reads settings once when an execution begins preparation. That +snapshot governs instruction reservation, compatibility and launch. Already +preparing or running work retains its configuration. A changed settings +revision or resolved credential version retires an idle incompatible warm +process using existing ownership checks. Conversation compatibility is +separate, so a safe replacement can resume the saved conversation. Recovery +never replays an uncertain turn merely to apply new export settings. + +Missing, disabled, invalid or unavailable configuration runs without company +export and reports a sanitized availability diagnostic where relevant. Native +export rejection/timeouts do not determine task success; exporter deadlines +are three seconds. Destination calls do not occur in usage, event-append, +tool-acknowledgment or settlement paths. + +## Credentials and access + +Only board users with Manage connections permission can read or change this +configuration. Agents cannot retrieve it through the settings API. The destination stores no credential +values. Authentication headers live in one encrypted Paperclip-managed company +secret, bound to the system consumer company-observability. Updates rotate that +secret; removal uses the secret service to retire it. Braintrust project +routing is derived from the current project ID, independently of the API key. + +Reads return configured status and header names. Blank existing custom header +values retain the saved secret at the same HTTP origin; a change of scheme, host, +or port requires every retained header value to be entered again. New/renamed headers require values. Removing a +row removes that header. API-key replacement is explicit, and Cancel clears +entered secrets. Revisions prevent another administrator's edit being silently +overwritten. Credential values are excluded from task environment manifests, +run/config snapshots, argv, activity records, recovery configuration and query +cache data. Export credentials are necessarily accessible to the exporting +harness itself. Claude Code supplies these headers through its process environment, +which its tools can inherit or inspect. Codex can read its managed configuration. +This feature does not isolate export credentials from agent code. Use a credential +limited to telemetry ingestion for the selected destination; do not supply a broad +organization administration key. A user-operated collector can hold upstream +credentials when a deployment requires a separate trust boundary. + +## Connection tests + +Saving never contacts the destination. Test saved settings sends one synthetic +root span and a correlated log with a unique marker from the control plane, +even when agent export is disabled. The receiver receives no production task +content. Both requests share a ten-second deadline, cannot redirect, and use +the deployment's guarded HTTP policy with DNS pinning and response-size limits. +Receipts contain sanitized HTTP/signal results and no receiver body text. +A receipt is retained only for the revision that was tested. + +“Test payload accepted” means the server's synthetic payload was accepted. It +does not prove agent export, remote network reachability, downstream indexing, +or that a provider UI rendered the records. Harness exporters use their own +network clients under the execution environment's policy; the server's DNS +pinning applies only to the explicit control-plane test. Verify real tasks and +the destination's records before relying on a deployment. + +## Verification + +The focused suites cover schema bounds, concurrent saves, secret/header +lifecycle, authorization, protobuf transport and rejection, private-network +policy, timeout, revision fencing, environment projections, private native +configuration, warm compatibility, and UI conflict/company-switch behavior. +Storybook IDs agent-observability-01-settings, agent-observability-02-states and +agent-observability-03-journey use fake credentials and production components. +Real local/remote native export evidence must be reported separately from +these automated and synthetic checks. diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 40fb9dcb733..138f118b5a1 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -556,3 +556,10 @@ retries cleanup. This event preserves the run outcome and does not claim a file save; `instruction_save` remains authoritative for collection. The event contains no raw exception, host path, file contents, or lock-owner metadata. Failure to write the warning must not replace the provider outcome or stop lease release. + +## Agent observability availability + +`agent_observability.unavailable` is one warning for a preparation snapshot that +cannot apply company export settings. Its payload contains only a closed reason +and the settings revision. It carries no endpoint, credential, secret reference +or resolver error. Recording the diagnostic never contacts a destination. diff --git a/maps/agent-access-map/companies.md b/maps/agent-access-map/companies.md index 0e3eabedc4b..eddecd4c725 100644 --- a/maps/agent-access-map/companies.md +++ b/maps/agent-access-map/companies.md @@ -183,3 +183,25 @@ success receipt, timeout and recorded usage. Gaps: No agent configuration/test path. An acknowledgement is not completed task work. Live native, provider and agent access remain unverified. + +## `agent-observability` + +Review: `baseline`. + +API: `present` — representative operation IDs: + +- `GET /api/companies/{companyId}/observability`. +- `PUT /api/companies/{companyId}/observability`. +- `DELETE /api/companies/{companyId}/observability`. +- `POST /api/companies/{companyId}/observability/test`. + +Agent access: `operator-only` — configuration and tests require board context. + +Policy: Company access and Manage connections permission; non-managing board readers receive no settings. +Credentials are write-only and stored through company secret bindings. + +Evidence: `source-only` — [routes](../../server/src/routes/agent-observability.ts). + +Verification: `not-run` — exercise board management, restricted board, agent denial, and cross-company denial. + +Gaps: Direct agent management and independent personal harness enrollment are outside v1. diff --git a/maps/feature-map/companies.md b/maps/feature-map/companies.md index 12b7caac314..e54b082d8df 100644 --- a/maps/feature-map/companies.md +++ b/maps/feature-map/companies.md @@ -16,6 +16,7 @@ Agent access: [companion](../agent-access-map/companies.md). - `identity`: update company identity and configured settings. - `decision-models`: configure and test the company's decision model, then inspect invocation history and recorded usage. - `fast-responses`: configure and test the experimental fast-response service, then inspect acknowledgement history, failures, and recorded usage. +- `agent-observability`: configure one company OTLP destination for Codex and Claude Code, test synthetic delivery, and disable export. - `archive`: archive and restore through the available controls without confusing archival with deletion. - `portability`: preview a company package and inspect imported agents, projects, skills, and conflicts. @@ -107,3 +108,18 @@ Sources: Source: [CompanyImport.tsx](../../ui/src/pages/CompanyImport.tsx), [Com ### Documentation scope: Company and instance administration Core administration/import/export/operator workflows are mapped; cloud/self-hosted differences, migration/restore compatibility, and platform-specific service variants need deeper recipes. + +### `agent-observability` + +Use Company Settings → General → Agent observability. Select Braintrust or Custom OTLP, +save credentials and capture settings, then test the saved destination. Saving is offline; +the test originates at the control plane and does not prove remote runner reachability. + +Automated: shared validation, encrypted configuration service, receiver tests, runtime projection, +and production settings stories cover their respective boundaries. Provider delivery requires a real run. + +Manual: configure a disposable company, reload, test, run Codex and Claude Code locally and remotely, +inspect their unique markers at the destination, then disable and verify a new marker is absent. + +Storybook verification: `agent-observability-01-settings`, `agent-observability-02-states`, +and `agent-observability-03-journey` mount production subjects with simulated APIs/providers. diff --git a/packages/adapter-utils/package.json b/packages/adapter-utils/package.json index 8f424019ab8..5b1bc75980f 100644 --- a/packages/adapter-utils/package.json +++ b/packages/adapter-utils/package.json @@ -45,7 +45,8 @@ "dependencies": { "@paperclipai/shared": "workspace:*", "acpx": "0.12.0", - "picocolors": "^1.1.1" + "picocolors": "^1.1.1", + "smol-toml": "^1.9.1" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 7ef0e510688..5cc2713bdc2 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { parse as parseToml } from "smol-toml"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createRuntimeStore, type AcpRuntimeOptions, type AcpSessionRecord, type AcpSessionStore } from "acpx/runtime"; import type { AdapterExecutionContext, AdapterRuntimeMcpAccess } from "@paperclipai/adapter-utils"; @@ -182,6 +183,7 @@ async function runExecutor( executionTransport?: Record; authToken?: string; agentIdentity?: AdapterExecutionContext["agentIdentity"]; + observability?: AdapterExecutionContext["observability"]; executionTarget?: Record; runtimeMcp?: AdapterRuntimeMcpAccess; prepareRemoteManagedHome?: AcpxEngineExecutorOptions["prepareRemoteManagedHome"]; @@ -229,6 +231,7 @@ async function runExecutor( executionTransport: options.executionTransport, authToken: options.authToken, agentIdentity: options.agentIdentity, + observability: options.observability, executionTarget: options.executionTarget, runtimeMcp: options.runtimeMcp, startupTraceContext: options.startupTraceContext, @@ -596,6 +599,66 @@ describe("shared ACPX engine runtime behavior", () => { expect(turnStartedBeforeProcessIdentity).toBe(false); }); + it("keeps company export config outside ACPX Codex argv and caller-owned home files", async () => { + const root = await makeTempRoot(); + const home = path.join(root, "caller-codex-home"); + await fs.mkdir(home, { recursive: true }); + await fs.writeFile(path.join(home, "config.toml"), 'model = "caller-model"\n'); + const { logs, meta } = await runExecutor({ agent: "codex", agentCommand: "node ./fake-acp.js", cwd: root, stateDir: path.join(root, "state"), env: { CODEX_HOME: home } }, { + observability: { kind: "codex", compatibilityKey: "fixture-revision", otelConfig: { exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: { authorization: "Bearer export-secret-canary" } } } }, environment: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000" } }, + }); + expect(await fs.readFile(path.join(home, "config.toml"), "utf8")).toBe('model = "caller-model"\n'); + expect(logs.some(row => row.text.includes("caller-owned Codex home"))).toBe(true); + expect(JSON.stringify(meta)).not.toContain("export-secret-canary"); + }); + + it.each([true, false])("applies company export to a server-managed Codex credential home (enabled=%s)", async (enabled) => { + const root = await makeTempRoot(); + const home = path.join(root, "server-credential-projection", "provider"); + const original = 'model = "managed-model"\n[model_providers.paperclip]\nname = "Managed provider"\n[mcp_servers.fixture]\nurl = "https://tools.example.test/mcp"\n'; + await fs.mkdir(home, { recursive: true }); + await fs.writeFile(path.join(home, "config.toml"), original); + await fs.writeFile(path.join(home, "auth.json"), '{"OPENAI_API_KEY":"fake-managed-provider-key"}'); + const otelConfig = enabled ? { + log_user_prompt: true, + exporter: { "otlp-http": { endpoint: "https://collector.example.test/v1/logs", protocol: "binary", headers: { authorization: "Bearer export-secret-canary" } } }, + } : { exporter: "none", trace_exporter: "none", metrics_exporter: "none", log_user_prompt: false }; + const run = await runExecutor({ + agent: "codex", agentCommand: "node ./fake-acp.js", cwd: root, stateDir: path.join(root, "state"), + env: { CODEX_HOME: home }, managedAiConnection: { provider: "openai", method: "api_key", identity: "server-resolved-credential" }, + paperclipRuntimeSkills: [], paperclipSkillSync: { desiredSkills: [] }, + }, { observability: { kind: "codex", compatibilityKey: enabled ? "enabled" : "disabled", otelConfig, environment: {} } }); + expect(run.result.exitCode).toBe(0); + const env = (run.sessionInputs[0]!.sessionOptions as { env: Record }).env; + expect(env.CODEX_HOME).not.toBe(home); + const config = parseToml(await fs.readFile(path.join(env.CODEX_HOME!, "config.toml"), "utf8")); + expect(config.otel).toEqual(otelConfig); + expect(config.model).toBe("managed-model"); + expect(config.model_providers).toEqual({ paperclip: { name: "Managed provider" } }); + expect(config.mcp_servers).toEqual({ fixture: { url: "https://tools.example.test/mcp" } }); + expect(await fs.readFile(path.join(home, "config.toml"), "utf8")).toBe(original); + expect(await fs.realpath(path.join(env.CODEX_HOME!, "auth.json"))).toBe(await fs.realpath(path.join(home, "auth.json"))); + expect(JSON.stringify(run.meta)).not.toContain("export-secret-canary"); + expect(run.logs.some(row => row.text.includes("caller-owned Codex home"))).toBe(false); + }); + + it("changes the ACPX Codex configuration home while resuming the same conversation after export changes", async () => { + const root = await makeTempRoot(); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "paperclip")); + vi.stubEnv("CODEX_HOME", path.join(root, "source")); + try { + const config = { agent: "codex", agentCommand: "node ./fake-acp.js", cwd: root, stateDir: path.join(root, "state"), paperclipRuntimeSkills: [], paperclipSkillSync: { desiredSkills: [] } }; + const launch = (key: string) => ({ kind: "codex" as const, compatibilityKey: key, otelConfig: { exporter: "none", trace_exporter: "none", metrics_exporter: "none" }, environment: {} }); + const first = await runExecutor(config, { observability: launch("first") }); + const second = await runExecutor(config, { observability: launch("second"), runtime: { sessionParams: first.result.sessionParams } }); + const firstEnv = (first.sessionInputs[0]?.sessionOptions as { env: Record }).env; + const secondEnv = (second.sessionInputs[0]?.sessionOptions as { env: Record }).env; + expect(firstEnv.CODEX_HOME).not.toBe(secondEnv.CODEX_HOME); + expect(second.sessionInputs[0]?.resumeSessionId).toBe(first.result.sessionId); + expect(second.result.sessionParams?.configFingerprint).toBe(first.result.sessionParams?.configFingerprint); + } finally { vi.unstubAllEnvs(); } + }); + it("sets Codex model, effort, and fast mode through CODEX_CONFIG without session config calls", async () => { const { configOptions, meta, receipts, result } = await runExecutor({ agent: "codex", diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index a5032f12695..c856bee0193 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -1,4 +1,6 @@ import { isOpenRouterUrl, openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution"; +import { prepareAcpxCodexObservabilityHome } from "../codex-observability-home.js"; +import { AGENT_OBSERVABILITY_ENV_KEYS, applyObservabilityEnvironment } from "../agent-observability.js"; import { cancellableSandboxStartup } from "./startup-cancellation.js"; import { withAdapterExecutionPhase, type AdapterExecutionPhase } from "../execution-phase.js"; import fs from "node:fs/promises"; @@ -232,6 +234,7 @@ type AcpxRuntimeFactory = (options: PaperclipAcpRuntimeOptions) => AcpRuntime; * in-sandbox staged workspace + home, which this cache reuses. */ export interface StagedRuntimeCacheEntry { + observabilityCompatibilityKey?: string; stagedRuntime: PreparedAdapterExecutionTargetRuntime; /** * The env keys the per-adapter managed-home seam mutated when it staged (e.g. @@ -1315,6 +1318,7 @@ async function prepareCodexSkillRuntime(input: { env: Record; moduleDir: string; onLog: AdapterExecutionContext["onLog"]; + observability?: AdapterExecutionContext["observability"]; // Step-timing seam: threaded from `buildRuntime` so the nested // `skills.reconcile` boundary (step 3) can emit its own `run.startup.step` // event at its call-site. Both optional — a caller without an event sink or @@ -1353,6 +1357,17 @@ async function prepareCodexSkillRuntime(input: { apiKey, onLog: input.onLog, }); + let launchCodexHome = effectiveCodexHome; + if (input.observability?.otelConfig) { + // Managed AI connections project a server-owned credential home into env. + // Only a caller override lacks this server-supplied connection attribution. + if (configuredCodexHome && !input.config.managedAiConnection) await input.onLog("stderr", "[paperclip] Company observability is unavailable for a caller-owned Codex home.\n"); + else { + try { + launchCodexHome = await prepareAcpxCodexObservabilityHome(effectiveCodexHome, input.observability, input.agentId); + } catch { await input.onLog("stderr", "[paperclip] Company observability configuration could not be prepared; execution continues without company export.\n"); } + } + } const { allSkills, selectedSkills, desiredSkillNames } = await resolveSelectedRuntimeSkills(input.config, input.moduleDir); const skillSetKey = await buildSkillSetKey({ skills: selectedSkills, label: "codex" }); const skillsHome = path.join(effectiveCodexHome, "skills"); @@ -1396,7 +1411,7 @@ async function prepareCodexSkillRuntime(input: { } await writeManagedCodexSkillsManifest(skillsHome, selectedSkills.map((entry) => entry.runtimeName)); - input.env.CODEX_HOME = effectiveCodexHome; + input.env.CODEX_HOME = launchCodexHome; return { identity: { @@ -2078,6 +2093,9 @@ async function buildRuntime(input: { resolvedAdapterEnv.ANTHROPIC_CUSTOM_HEADERS = env.ANTHROPIC_CUSTOM_HEADERS; } if (authToken) env.PAPERCLIP_API_KEY = authToken; + if (input.ctx.observability) { + applyObservabilityEnvironment(env, input.ctx.observability); + } // For the claude agent, set model via ANTHROPIC_MODEL at startup rather than // via session/set_config_option — the ACP server's set_config_option handler // validates the value against its internal available-models list and rejects @@ -2106,7 +2124,7 @@ async function buildRuntime(input: { "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY", // Preserve Codex's default secret-name exclusions. The short-lived // Paperclip API token is required by the agent skill's Bash/curl calls. - ...Object.keys(env).filter(key => key === "PAPERCLIP_API_KEY" || !/key|secret|token/i.test(key)), + ...Object.keys(env).filter(key => !(AGENT_OBSERVABILITY_ENV_KEYS as readonly string[]).includes(key) && (key === "PAPERCLIP_API_KEY" || !/key|secret|token/i.test(key))), ])] } : {}), requestedModel, requestedThinkingEffort, @@ -2167,6 +2185,7 @@ async function buildRuntime(input: { agentId: agent.id, config, env, + observability: input.ctx.observability, moduleDir: input.engine.moduleDir, onLog: input.ctx.onLog, onEvent: input.ctx.onEvent, @@ -2391,6 +2410,7 @@ async function buildRuntime(input: { target: remoteTarget, env, isCompatibleResume, + observabilityCompatibilityKey: input.ctx.observability?.compatibilityKey, stage: (assets) => stageAcpRemoteRuntime({ runId, @@ -3670,10 +3690,12 @@ function saveStagedRuntimeAfterCleanTurn(input: { handles: Map; prepared: AcpxPreparedRuntime; now: number; + observabilityCompatibilityKey?: string; }) { const { prepared } = input; if (!prepared.stagedRuntime || prepared.remoteStagingEnvDelta === null) return; input.handles.set(prepared.sessionKey, { + observabilityCompatibilityKey: input.observabilityCompatibilityKey, stagedRuntime: prepared.stagedRuntime, envDelta: prepared.remoteStagingEnvDelta, teardown: prepared.remoteManagedHomeTeardown, @@ -4403,7 +4425,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { // the reused runtime cannot expire under its own timer while this run uses // it (today's `clearWarmHandleTimer(cached)` after the reuse decision). let cached = canResume ? hostStore.borrow(prepared.sessionKey) : undefined; - if (cached && (ctx.context.refreshTools === true + if (cached && (cached.observabilityCompatibilityKey !== ctx.observability?.compatibilityKey || ctx.context.refreshTools === true || asString(previousParams.mcpFingerprint, "") !== shortHash(prepared.mcpIdentity) // MCP bearer credentials are run-scoped, even for an unchanged catalog. || prepared.mcpServers.length > 0)) { @@ -5688,7 +5710,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { settleReuse: (slots, decision) => timedPhase("settle_reuse", async () => { if (!slots.has("staged_runtime")) return; if (decision.kind === "save" && decision.savedId === "staged_runtime") { - saveStagedRuntimeAfterCleanTurn({ handles: stagedRuntimes, prepared, now: now() }); + saveStagedRuntimeAfterCleanTurn({ handles: stagedRuntimes, prepared, now: now(), observabilityCompatibilityKey: ctx.observability?.compatibilityKey }); return; } await discardStagedRuntime({ handles: stagedRuntimes, prepared }); diff --git a/packages/adapter-utils/src/acpx-engine/run-site-host.ts b/packages/adapter-utils/src/acpx-engine/run-site-host.ts index e2fa028ca88..dd44c3bf558 100644 --- a/packages/adapter-utils/src/acpx-engine/run-site-host.ts +++ b/packages/adapter-utils/src/acpx-engine/run-site-host.ts @@ -52,6 +52,7 @@ export interface ChildStderrState { * time and its own optional idle timer, which the store reads and writes. */ export interface RuntimeCacheEntry { + observabilityCompatibilityKey?: string; runtime: AcpRuntime; handle: AcpRuntimeHandle; childStderrState: ChildStderrState; diff --git a/packages/adapter-utils/src/acpx-engine/run-site-sandbox.test.ts b/packages/adapter-utils/src/acpx-engine/run-site-sandbox.test.ts index ebbab2cb607..bee3ab8ff68 100644 --- a/packages/adapter-utils/src/acpx-engine/run-site-sandbox.test.ts +++ b/packages/adapter-utils/src/acpx-engine/run-site-sandbox.test.ts @@ -7,7 +7,7 @@ // creates the runtime), the synchronous plan, and Amendment B — no run-scoped // contribution or bridge token escapes into the ledger or the reuse payload. -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createSandboxRunSite, type ManagedHomeSeamResult, @@ -290,6 +290,21 @@ describe("sandbox run site", () => { }); }); + it("replaces staged exporter configuration without changing the resumed conversation key", async () => { + const dispose = vi.fn(async () => {}); + const cachedRuntime = makeStagedRuntime("old-exporter"); + const stagedRuntimes = new Map([["paperclip:c:a:t:fp", { + stagedRuntime: cachedRuntime, envDelta: { CODEX_HOME: "/remote/old-home" }, + teardown: null, dispose, lastUsedAt: 500, observabilityCompatibilityKey: "old-revision", + }]]); + const { site, env } = makeSite({ stagedRuntimes, isCompatibleResume: true, observabilityCompatibilityKey: "new-revision" }); + await site.placeWorkspace(makeContext("paperclip:c:a:t:fp")); + expect(dispose).toHaveBeenCalledOnce(); + expect(site.staged?.reused).toBe(false); + expect(site.staged?.stagedRuntime).not.toBe(cachedRuntime); + expect(env.CODEX_HOME).not.toBe("/remote/old-home"); + }); + it("test_bridge_contribution_is_not_retained_by_site_ledger_or_reuse_payload", async () => { const { site, registered } = makeSite(); await site.placeWorkspace(makeContext("s")); diff --git a/packages/adapter-utils/src/acpx-engine/run-site-sandbox.ts b/packages/adapter-utils/src/acpx-engine/run-site-sandbox.ts index 73f6a86fe54..fb286b189ef 100644 --- a/packages/adapter-utils/src/acpx-engine/run-site-sandbox.ts +++ b/packages/adapter-utils/src/acpx-engine/run-site-sandbox.ts @@ -51,6 +51,7 @@ import type { WorkspaceRestoreOutcome } from "../workspace-restore-merge.js"; * none — the run-start sweep is the only eviction path. */ export interface StagedRuntimeStoreEntry { + observabilityCompatibilityKey?: string; stagedRuntime: PreparedAdapterExecutionTargetRuntime; envDelta: Record; teardown: (() => Promise) | null; @@ -111,6 +112,8 @@ export interface SandboxRunSiteOptions { * compatible resume may reuse the already-staged runtime. */ readonly isCompatibleResume: boolean; + /** Process configuration compatibility; deliberately separate from conversation identity. */ + readonly observabilityCompatibilityKey?: string; /** Ship the workspace (and, via the seam, the managed home) into the sandbox. */ readonly stage: StageWorkspace; /** @@ -233,7 +236,7 @@ export function createSandboxRunSite(options: SandboxRunSiteOptions): SandboxRun const sessionKey = context.sessionKey; const lease = await withStagingLease(options.stagingLocks, sessionKey, async (): Promise => { const cached = options.isCompatibleResume ? store.borrow(sessionKey) : undefined; - if (cached) { + if (cached && cached.observabilityCompatibilityKey === options.observabilityCompatibilityKey) { // Reuse the already-staged in-sandbox workspace and managed home. // Re-apply the env keys the seam repointed and reuse the per-run // copy-back so the copy-back cadence stays exactly per run. diff --git a/packages/adapter-utils/src/agent-observability.test.ts b/packages/adapter-utils/src/agent-observability.test.ts new file mode 100644 index 00000000000..cb85c54eef9 --- /dev/null +++ b/packages/adapter-utils/src/agent-observability.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, it } from "vitest"; +import { lstat, mkdtemp, readFile, readlink, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse } from "smol-toml"; +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE } from "@paperclipai/shared"; +import { buildProviderObservabilityLaunch, applyObservabilityEnvironment, observabilitySecretValues } from "./agent-observability.js"; +import { prepareCodexObservabilityProfile, replaceCodexProfileArgs, selectedCodexProfile, disableCodexObservabilityArgs } from "./codex-observability-profile.js"; +import { buildInvocationEnvForLogs } from "./server-utils.js"; +import { prepareAcpxCodexObservabilityHome } from "./codex-observability-home.js"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const input = { companyId: "company", agentId: "agent", revision: "revision", compatibilityKey: "version", enabled: true, + capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, logsEndpoint: "https://collector.example/v1/logs", tracesEndpoint: "https://collector.example/v1/traces", headers: { authorization: "Bearer a,b=c% d" } }; +it("encodes headers losslessly and clears ambient exporter conflicts without changing host state", () => { + const launch = buildProviderObservabilityLaunch({ ...input, kind: "claude" }); + const env = { OTEL_EXPORTER_OTLP_HEADERS: "ambient", OTEL_EXPORTER_OTLP_ENDPOINT: "https://wrong", KEEP: "ok" }; + applyObservabilityEnvironment(env, launch); + expect(env).not.toHaveProperty("OTEL_EXPORTER_OTLP_HEADERS"); + expect(env).toHaveProperty("KEEP", "ok"); + const encoded = launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS; + expect(decodeURIComponent(encoded.slice(encoded.indexOf("=") + 1))).toBe(input.headers.authorization); + expect(observabilitySecretValues(launch)).toContain(input.headers.authorization); + expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb"); +}); +it("preserves literal percent sequences through the qualified Codex exporter's header decoder", () => { + const original = "Bearer a,b=%26 c%"; + const launch = buildProviderObservabilityLaunch({ ...input, kind: "codex", headers: { authorization: original } }); + const exporter = launch.otelConfig!.exporter as { "otlp-http": { headers: Record } }; + expect(decodeURIComponent(exporter["otlp-http"].headers.authorization)).toBe(original); + expect(observabilitySecretValues(launch)).toContain(original); + expect(observabilitySecretValues(launch)).toContain(original.slice(7)); +}); +it("replaces a compact explicit profile without leaving competing profile flags", () => { + const args = ["exec", "-preview", "-"]; + expect(selectedCodexProfile(args)).toBe("review"); + replaceCodexProfileArgs(args, "generated"); + expect(args).toEqual(["--profile", "generated", "exec", "-"]); +}); +it("keeps an explicit source profile while disabling export after preparation fails", () => { + const args = ["--profile", "review", "exec", "-"]; + disableCodexObservabilityArgs(args); + expect(selectedCodexProfile(args)).toBe("review"); + expect(args).toContain('otel.exporter="none"'); + expect(args).toContain('otel.trace_exporter="none"'); + expect(args).toContain('otel.metrics_exporter="none"'); +}); +it("creates invocation-owned profiles, preserves selected profile settings, and restores by deleting only its own file", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-otel-profile-")); roots.push(root); + await writeFile(join(root, "review.config.toml"), 'model = "chosen-model"\n[otel]\nexporter = "none"\n'); + const launch = buildProviderObservabilityLaunch({ ...input, kind: "codex" }); + const a = await prepareCodexObservabilityProfile({ home: root, runId: "run-a", sourceProfile: "review", launch }); + const b = await prepareCodexObservabilityProfile({ home: root, runId: "run-b", launch }); + const parsed = parse(await readFile(join(root, a.filename), "utf8")); + expect(parsed.model).toBe("chosen-model"); + expect(parsed.otel).toMatchObject({ log_agent_responses: true, tool_result: { max_bytes: 65536 } }); + expect((await stat(join(root, a.filename))).mode & 0o777).toBe(0o600); + const args = ["exec", "--profile=review", "resume", "session", "-"]; + replaceCodexProfileArgs(args, a.name); + expect(args).toEqual(["--profile", a.name, "exec", "resume", "session", "-"]); + await a.cleanup(); + expect(await readFile(join(root, b.filename), "utf8")).toContain("otel"); + expect(await readFile(join(root, "review.config.toml"), "utf8")).toContain('exporter = "none"'); +}); +it("isolates overlapping ACPX export settings while sharing the saved provider conversation", async () => { + const home = await mkdtemp(join(tmpdir(), "paperclip-acpx-otel-")); roots.push(home); + await writeFile(join(home, "config.toml"), 'model = "retained-model"\n'); + const first = buildProviderObservabilityLaunch({ ...input, kind: "codex", compatibilityKey: "first" }); + const second = buildProviderObservabilityLaunch({ ...input, kind: "codex", compatibilityKey: "second", enabled: false }); + const otherAgent = buildProviderObservabilityLaunch({ ...input, kind: "codex", compatibilityKey: "first", agentId: "other-agent" }); + const [a, b, c] = await Promise.all([ + prepareAcpxCodexObservabilityHome(home, first, input.agentId), + prepareAcpxCodexObservabilityHome(home, second, input.agentId), + prepareAcpxCodexObservabilityHome(home, otherAgent, "other-agent"), + ]); + expect(a).not.toBe(c); + expect(parse(await readFile(join(a, "config.toml"), "utf8")).otel).toMatchObject({ span_attributes: { "paperclip.agent.id": input.agentId } }); + expect(parse(await readFile(join(c, "config.toml"), "utf8")).otel).toMatchObject({ span_attributes: { "paperclip.agent.id": "other-agent" } }); + await writeFile(join(a, "sessions", "saved-conversation.jsonl"), "conversation checkpoint"); + expect(await readFile(join(b, "sessions", "saved-conversation.jsonl"), "utf8")).toBe("conversation checkpoint"); + expect(parse(await readFile(join(a, "config.toml"), "utf8")).otel).toMatchObject({ log_agent_responses: true }); + expect(parse(await readFile(join(b, "config.toml"), "utf8")).otel).toMatchObject({ exporter: "none" }); + expect(parse(await readFile(join(a, "config.toml"), "utf8")).model).toBe("retained-model"); + expect(await readFile(join(home, "config.toml"), "utf8")).toBe('model = "retained-model"\n'); + expect((await stat(a)).mode & 0o777).toBe(0o700); + expect((await stat(join(a, "config.toml"))).mode & 0o777).toBe(0o600); +}); + +it.each([true, false])("preserves refreshed Codex credentials across repeated and replacement preparations with export %s", async enabled => { + const home = await mkdtemp(join(tmpdir(), "paperclip-acpx-auth-")); roots.push(home); + await writeFile(join(home, "auth.json"), JSON.stringify({ tokens: { refresh_token: "old-fixture-token" } }), { mode: 0o600 }); + const launch = buildProviderObservabilityLaunch({ ...input, enabled, kind: "codex" }); + const first = await prepareAcpxCodexObservabilityHome(home, launch, input.agentId); + expect((await lstat(join(first, "auth.json"))).isSymbolicLink()).toBe(true); + expect(await readlink(join(first, "auth.json"))).toBe(join(home, "auth.json")); + await writeFile(join(first, "auth.json"), JSON.stringify({ tokens: { refresh_token: "refreshed-fixture-token" } })); + expect(JSON.parse(await readFile(join(home, "auth.json"), "utf8")).tokens.refresh_token).toBe("refreshed-fixture-token"); + const repeated = await prepareAcpxCodexObservabilityHome(home, launch, input.agentId); + const replacement = await prepareAcpxCodexObservabilityHome(home, { ...launch, compatibilityKey: "replacement" }, input.agentId); + for (const prepared of [repeated, replacement]) { + expect(JSON.parse(await readFile(join(prepared, "auth.json"), "utf8")).tokens.refresh_token).toBe("refreshed-fixture-token"); + } +}); diff --git a/packages/adapter-utils/src/agent-observability.ts b/packages/adapter-utils/src/agent-observability.ts new file mode 100644 index 00000000000..aab46b24457 --- /dev/null +++ b/packages/adapter-utils/src/agent-observability.ts @@ -0,0 +1,81 @@ +import type { AgentObservabilityCapture } from "@paperclipai/shared"; + +/** Secret-bearing, controller-owned launch material. Never persist or merge into adapter config. */ +export type ProviderObservabilityLaunch = { + kind: "codex" | "claude"; + compatibilityKey: string; + otelConfig?: Record; + environment: Readonly>; +}; + +export const AGENT_OBSERVABILITY_ENV_KEYS = [ + "CLAUDE_CODE_ENABLE_TELEMETRY", "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "ENABLE_ENHANCED_TELEMETRY_BETA", + "OTEL_LOGS_EXPORTER", "OTEL_TRACES_EXPORTER", "OTEL_METRICS_EXPORTER", "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG", + "OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT", "OTEL_LOG_RAW_API_BODIES", +] as const; + +export function encodeObservabilityHeaders(headers: Readonly>): string { + return Object.entries(headers).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join(","); +} + +export function applyObservabilityEnvironment(env: NodeJS.ProcessEnv, launch: ProviderObservabilityLaunch): void { + for (const name of AGENT_OBSERVABILITY_ENV_KEYS) delete env[name]; + Object.assign(env, launch.environment); +} + +export function observabilitySecretValues(launch: ProviderObservabilityLaunch | undefined): string[] { + if (!launch) return []; + const headers = ((launch.otelConfig?.exporter as Record | undefined)?.["otlp-http"] as { headers?: Record } | undefined)?.headers; + const encoded = launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS; + const decoded = (value: string) => { try { return decodeURIComponent(value); } catch { return value; } }; + const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").map(item => decoded(item.slice(item.indexOf("=") + 1)))] : [])]; + return [...new Set(values.flatMap(value => [value, value.replace(/^Bearer /i, ""), JSON.stringify(value).slice(1, -1)]).filter(Boolean))]; +} + +export function buildProviderObservabilityLaunch(input: { + kind: "codex" | "claude"; companyId: string; agentId: string; revision: string | null; + compatibilityKey: string; enabled: boolean; capture: AgentObservabilityCapture; + logsEndpoint?: string; tracesEndpoint?: string; headers?: Readonly>; +}): ProviderObservabilityLaunch { + const { enabled, capture, kind, compatibilityKey } = input; + const attributes = { "paperclip.company.id": input.companyId, "paperclip.agent.id": input.agentId, "paperclip.observability.revision": input.revision ?? "unconfigured" }; + const environment: Record = { + OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000", OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "3000", + }; + if (kind === "codex") { + // Codex passes these through opentelemetry-otlp 0.31's with_headers, which + // percent-decodes values. Encode once so literal percent sequences survive. + const headers = Object.fromEntries(Object.entries(input.headers ?? {}).map(([name, value]) => [name, encodeURIComponent(value)])); + const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers } } : "none"; + return { kind, compatibilityKey, environment, otelConfig: { + exporter: exporter(input.logsEndpoint), trace_exporter: exporter(input.tracesEndpoint), metrics_exporter: "none", + log_user_prompt: enabled && capture.userPrompts, log_agent_responses: enabled && capture.assistantResponses, + log_guardian_assessments: false, tool_result: { max_bytes: enabled && capture.toolContent ? 65536 : 0 }, + span_attributes: enabled ? attributes : {}, + } }; + } + Object.assign(environment, { + CLAUDE_CODE_ENABLE_TELEMETRY: enabled ? "1" : "0", + CLAUDE_CODE_ENHANCED_TELEMETRY_BETA: enabled ? "1" : "0", + OTEL_LOGS_EXPORTER: enabled ? "otlp" : "none", OTEL_TRACES_EXPORTER: enabled ? "otlp" : "none", OTEL_METRICS_EXPORTER: "none", + OTEL_LOG_USER_PROMPTS: enabled && capture.userPrompts ? "1" : "0", + OTEL_LOG_ASSISTANT_RESPONSES: enabled && capture.assistantResponses ? "1" : "0", + OTEL_LOG_TOOL_DETAILS: enabled && capture.toolDetails ? "1" : "0", + OTEL_LOG_TOOL_CONTENT: enabled && capture.toolContent ? "1" : "0", OTEL_LOG_RAW_API_BODIES: "0", + }); + if (enabled) Object.assign(environment, { + OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: "http/protobuf", OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf", + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: input.logsEndpoint, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: input.tracesEndpoint, + OTEL_EXPORTER_OTLP_LOGS_HEADERS: encodeObservabilityHeaders(input.headers ?? {}), + OTEL_EXPORTER_OTLP_TRACES_HEADERS: encodeObservabilityHeaders(input.headers ?? {}), + OTEL_RESOURCE_ATTRIBUTES: Object.entries(attributes).map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join(","), + }); + return { kind, compatibilityKey, environment }; +} diff --git a/packages/adapter-utils/src/codex-observability-home.ts b/packages/adapter-utils/src/codex-observability-home.ts new file mode 100644 index 00000000000..5acc38fc1b9 --- /dev/null +++ b/packages/adapter-utils/src/codex-observability-home.ts @@ -0,0 +1,58 @@ +import { chmod, mkdir, readFile, readlink, rename, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { parse, stringify } from "smol-toml"; +import type { ProviderObservabilityLaunch } from "./agent-observability.js"; + +/** ACPX starts app-server: its config must be private to this settings snapshot. + * Conversation files stay in the stable managed home; no export config is + * written there or included in the logical session fingerprint. + */ +export async function prepareAcpxCodexObservabilityHome(home: string, launch: ProviderObservabilityLaunch, agentId: string): Promise { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(agentId) || !/^[A-Za-z0-9_-]{1,128}$/.test(launch.compatibilityKey) || launch.kind !== "codex" || !launch.otelConfig) { + throw new Error("Invalid Codex observability launch"); + } + const parent = path.join(home, "paperclip-observability"); + // Subscription agents share the company home and compatibility key, but + // their resource attributes differ. Never let their configs share a file. + const owner = path.join(parent, agentId); + const target = path.join(owner, launch.compatibilityKey); + await mkdir(target, { recursive: true, mode: 0o700 }); + await chmod(parent, 0o700); await chmod(owner, 0o700); await chmod(target, 0o700); + async function atomicFile(name: string, contents: string | Buffer) { + const temporary = path.join(target, `.${name}-${randomUUID()}.tmp`); + try { + await writeFile(temporary, contents, { flag: "wx", mode: 0o600 }); + await rename(temporary, path.join(target, name)); + await chmod(path.join(target, name), 0o600); + } finally { await rm(temporary, { force: true }); } + } + // Preserve the existing managed credential link. Codex refreshes this file + // in place; a launch-owned copy would lose the rotated token on preparation. + const authLink = path.join(target, "auth.json"); + const temporaryAuthLink = path.join(target, `.auth-${randomUUID()}.tmp`); + try { + await symlink(path.join(home, "auth.json"), temporaryAuthLink, "file"); + await rename(temporaryAuthLink, authLink); + } finally { await rm(temporaryAuthLink, { force: true }); } + for (const name of ["config.json", "instructions.md"]) { + try { await atomicFile(name, await readFile(path.join(home, name))); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + } + for (const name of ["sessions", "archived_sessions", "skills"]) { + const source = path.join(home, name); + await mkdir(source, { recursive: true }); + const link = path.join(target, name); + try { await symlink(source, link, "dir"); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST" || await readlink(link) !== source) throw error; + } + } + const source = await readFile(path.join(home, "config.toml"), "utf8").catch(error => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return ""; throw error; + }); + const current = parse(source); + const features = current.features && typeof current.features === "object" && !Array.isArray(current.features) ? current.features : {}; + await atomicFile("config.toml", stringify({ ...current, otel: launch.otelConfig, features: { ...features, shell_snapshot: false } })); + return target; +} diff --git a/packages/adapter-utils/src/codex-observability-profile.ts b/packages/adapter-utils/src/codex-observability-profile.ts new file mode 100644 index 00000000000..6937443e9f3 --- /dev/null +++ b/packages/adapter-utils/src/codex-observability-profile.ts @@ -0,0 +1,48 @@ +import { chmod, lstat, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { parse, stringify } from "smol-toml"; +import type { ProviderObservabilityLaunch } from "./agent-observability.js"; + +export function selectedCodexProfile(args: readonly string[]): string | undefined { + const names: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === "--profile" || args[i] === "-p") names.push(args[++i] ?? ""); + else if (args[i].startsWith("--profile=")) names.push(args[i].slice(10)); + else if (args[i].startsWith("-p") && !args[i].startsWith("--")) names.push(args[i].slice(2)); + } + if (names.length > 1 || names.some(name => !/^[A-Za-z0-9_-]+$/.test(name))) throw new Error("Invalid Codex profile selection"); + return names[0]; +} + +export function replaceCodexProfileArgs(args: string[], profile: string): void { + for (let i = args.length - 1; i >= 0; i--) { + if (args[i].startsWith("--profile=") || args[i].startsWith("-p") && args[i] !== "-p" && !args[i].startsWith("--")) args.splice(i, 1); + else if (args[i] === "--profile" || args[i] === "-p") args.splice(i, 2); + } + args.unshift("--profile", profile); +} + +/** Non-secret fail-open overrides preserve the selected profile's other settings. */ +export function disableCodexObservabilityArgs(args: string[]): void { + args.unshift(...["exporter", "trace_exporter", "metrics_exporter"].flatMap(key => ["-c", `otel.${key}="none"`])); +} + +export async function prepareCodexObservabilityProfile(input: { + home: string; runId: string; launch: ProviderObservabilityLaunch; sourceProfile?: string; +}) { + if (!/^[A-Za-z0-9_-]+$/.test(input.runId)) throw new Error("Invalid observability run identity"); + const name = `paperclip-observability-${input.runId}`; + const filename = `${name}.config.toml`; + const file = path.join(input.home, filename); + let source: Record = {}; + if (input.sourceProfile) { + if (!/^[A-Za-z0-9_-]+$/.test(input.sourceProfile)) throw new Error("Invalid source profile"); + const sourcePath = path.join(input.home, `${input.sourceProfile}.config.toml`); + const stat = await lstat(sourcePath); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("Source profile must be a regular file"); + source = parse(await readFile(sourcePath, "utf8")); + } + await writeFile(file, stringify({ ...source, otel: input.launch.otelConfig ?? { exporter: "none", trace_exporter: "none", metrics_exporter: "none" } }), { flag: "wx", mode: 0o600 }); + await chmod(file, 0o600); + return { name, filename, cleanup: () => rm(file, { force: true }) }; +} diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 5f4b5ab0a23..6091b8580b4 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -3198,6 +3198,9 @@ export function buildInvocationEnvForLogs( redactCommandTextForLogs(resolvedCommand); } + for (const key of ["OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"]) { + if (key in merged) merged[key] = "[REDACTED]"; + } return redactEnvForLogs(merged); } diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index a9a4fdc97b8..37e6fc45a08 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -235,6 +235,8 @@ export interface AgentRuntimeIdentity { } export interface AdapterExecutionContext { + /** Company-owned exporter credentials. Transient; never persist or merge into config. */ + observability?: import("./agent-observability.js").ProviderObservabilityLaunch; /** Server-owned credentials. Never persist this object or merge it into config. */ agentIdentity?: AgentRuntimeIdentity; /** Synchronous, content-free diagnostic scope; never stop or collection authority. */ diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index e9dd7da6f5f..3029ddac5f1 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -1,4 +1,5 @@ import { isOpenRouterUrl, openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution"; +import { applyObservabilityEnvironment } from "@paperclipai/adapter-utils/agent-observability"; import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary"; import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint"; import fs from "node:fs/promises"; @@ -487,7 +488,12 @@ export async function execute(ctx: AdapterExecutionContext): Promise { vi.restoreAllMocks(); }); + it("stages only the invocation-owned private observability profile", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "codex-otel-stage-")); + let staged: string | undefined; + try { + const home = path.join(root, "home"); await fs.mkdir(home); + const own = "paperclip-observability-run-1.config.toml"; + const other = "paperclip-observability-run-2.config.toml"; + await fs.writeFile(path.join(home, own), '[otel]\nexporter = "none"\n', { mode: 0o600 }); + await fs.writeFile(path.join(home, other), "another-active-run", { mode: 0o600 }); + staged = await stageCodexHomeForSync(home, { runId: "run-1", additionalFiles: [own] }); + expect(await fs.readdir(staged)).toContain(own); + expect(await fs.readdir(staged)).not.toContain(other); + expect((await fs.stat(path.join(staged, own))).mode & 0o777).toBe(0o600); + expect((await fs.stat(staged)).mode & 0o777).toBe(0o700); + await expect(stageCodexHomeForSync(home, { additionalFiles: ["../escape.config.toml"] })).rejects.toThrow(); + await fs.chmod(path.join(home, own), 0o644); + await expect(stageCodexHomeForSync(home, { additionalFiles: [own] })).rejects.toThrow("private regular file"); + } finally { if (staged) await fs.rm(staged, { recursive: true, force: true }); await fs.rm(root, { recursive: true, force: true }); } + }); + // Builds a fake managed CODEX_HOME containing the full allowlist (with // `auth.json` as a symlink into a separate source-bytes file and a populated // `skills/` symlink, mirroring the real managed home) plus decoy runtime @@ -1095,6 +1121,122 @@ describe("stageCodexHomeForSync", () => { return { home, authBytes, skillBytes }; } + it.each([ + { extraArgs: ["--profile", "selected"], args: ["--profile", "ignored"], expectedModel: "selected-model" }, + { extraArgs: ["--full-auto"], args: ["--profile", "ignored"], expectedModel: undefined }, + { extraArgs: [], args: ["--profile", "legacy"], expectedModel: "legacy-model" }, + ])("uses effective CLI arguments when materializing an export profile: %j", async ({ extraArgs, args, expectedModel }) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-profile-policy-")); + const companyId = randomUUID(); + const home = path.join(root, "instances", "default", "companies", companyId, "codex-home"); + const captured = path.join(root, "captured-profile.json"); + try { + vi.stubEnv("PAPERCLIP_HOME", root); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "default"); + vi.stubEnv("CODEX_HOME", path.join(root, "source-home")); + await fs.mkdir(home, { recursive: true }); + for (const name of ["selected", "ignored", "legacy"]) { + await fs.writeFile(path.join(home, `${name}.config.toml`), `model = "${name}-model"\n`); + } + const command = path.join(root, "fake-codex"); + await fs.writeFile(command, `#!${process.execPath} +const fs = require("node:fs"), path = require("node:path"); +if (process.argv.includes("--version")) { console.log("codex-cli 0.160.0"); process.exit(0); } +process.stdin.resume(); +process.stdin.on("end", () => { + const argv = process.argv.slice(2), index = argv.indexOf("--profile"); + const profile = index < 0 ? null : fs.readFileSync(path.join(process.env.CODEX_HOME, argv[index + 1] + ".config.toml"), "utf8"); + fs.writeFileSync(${JSON.stringify(captured)}, JSON.stringify({ argv, profile })); + console.log(JSON.stringify({ type: "thread.started", thread_id: "fixture-session" })); + console.log(JSON.stringify({ type: "turn.completed", usage: { input_tokens: 1, output_tokens: 1 } })); +}); +`, { mode: 0o700 }); + const result = await executeCodex({ + runId: randomUUID(), agent: { id: randomUUID(), companyId, adapterType: "codex_local", name: "Profile fixture", adapterConfig: {} }, + runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null }, context: {}, config: { engine: "cli", command, cwd: root, extraArgs, args, + env: { CODEX_HOME: home, OPENAI_API_KEY: "fake-provider-key" }, paperclipSkillSync: { desiredSkills: [] } }, + observability: buildProviderObservabilityLaunch({ kind: "codex", companyId, agentId: "fixture-agent", revision: "revision", compatibilityKey: "fixture", + enabled: true, capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, + logsEndpoint: "https://collector.example/v1/logs", tracesEndpoint: "https://collector.example/v1/traces", headers: {} }), + onLog: async () => {}, onMeta: async () => {}, + }); + expect(result.exitCode).toBe(0); + const output = JSON.parse(await fs.readFile(captured, "utf8")); + expect(output.argv.filter((arg: string) => arg === "--profile")).toHaveLength(1); + expect(output.profile.match(/^model = "([^"]+)"$/m)?.[1]).toBe(expectedModel); + expect(output.profile).toContain("log_user_prompt = true"); + expect(output.profile).toContain("log_agent_responses = true"); + expect(output.argv).not.toContain("ignored"); + } finally { vi.unstubAllEnvs(); await fs.rm(root, { recursive: true, force: true }); } + }); + + it.each([false, true])("preserves remote conversations through real exporter restaging (subscription=%s)", async (mergeAuth) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-restage-")); + const stagedHomes: string[] = []; + try { + const home = path.join(root, "host-home"); + const workspace = path.join(root, "workspace"); + const remoteCwd = path.join(root, "remote"); + await Promise.all([home, workspace, remoteCwd].map(dir => fs.mkdir(dir, { recursive: true }))); + const auth = (date: string, marker: string) => JSON.stringify({ + tokens: { account_id: "fixture-account", access_token: marker, refresh_token: marker, id_token: marker }, last_refresh: date, + }); + const hostAuth = mergeAuth ? auth("2026-01-01T00:00:00Z", "host-token") : '{"OPENAI_API_KEY":"fake-api-key"}'; + const rotatedAuth = auth("2026-10-01T00:00:00Z", "rotated-remote-token"); + await fs.writeFile(path.join(home, "auth.json"), hostAuth); + const target = { + kind: "remote", transport: "sandbox", providerKey: "fixture", remoteCwd, + runner: { execute: async (input: { command: string; args?: string[]; cwd?: string; env?: Record; stdin?: string; timeoutMs?: number }) => + runChildProcess("codex-restage-fixture", input.command, input.args ?? [], { + cwd: input.cwd ?? root, env: input.env ?? {}, stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), graceSec: 1, onLog: async () => {}, + }) }, + } as const; + const stage = async (revision: string) => { + await fs.writeFile(path.join(home, "config.toml"), `[otel.exporter.otlp-http]\nendpoint = "https://${revision}.example.test/v1/logs"\nprotocol = "binary"\n`); + const staged = await stageCodexHomeForSync(home); + stagedHomes.push(staged); + expect(await fs.readdir(staged)).not.toContain("sessions"); + return prepareAdapterExecutionTargetRuntime({ + target, runId: revision, adapterKey: "codex-observability-fixture", workspaceLocalDir: workspace, + syncWorkspace: false, assets: [{ key: "home", localDir: staged, provision: buildCodexHomeRefreshProvision({ mergeAuth }) }], + }); + }; + const first = await stage("first"); + const remoteHome = first.assetDirs.home!; + for (const dir of ["sessions", "archived_sessions"]) { + await fs.mkdir(path.join(remoteHome, dir), { recursive: true }); + await fs.writeFile(path.join(remoteHome, dir, "conversation.jsonl"), `saved-${dir}`); + } + await fs.writeFile(path.join(remoteHome, "state_5.sqlite"), "native-session-index"); + await fs.writeFile(path.join(remoteHome, "config.json"), "stale-managed-config"); + if (mergeAuth) await fs.writeFile(path.join(remoteHome, "auth.json"), rotatedAuth); + const second = await stage("second"); + expect(second.assetDirs.home).toBe(remoteHome); + for (const dir of ["sessions", "archived_sessions"]) { + expect(await fs.readFile(path.join(remoteHome, dir, "conversation.jsonl"), "utf8")).toBe(`saved-${dir}`); + } + expect(await fs.readFile(path.join(remoteHome, "state_5.sqlite"), "utf8")).toBe("native-session-index"); + expect(await fs.readFile(path.join(remoteHome, "config.toml"), "utf8")).toContain('https://second.example.test/v1/logs'); + expect(await fs.readFile(path.join(remoteHome, "auth.json"), "utf8")).toBe(mergeAuth ? rotatedAuth : hostAuth); + await expect(fs.access(path.join(remoteHome, "config.json"))).rejects.toThrow(); + expect((await fs.stat(remoteHome)).mode & 0o777).toBe(0o700); + expect((await fs.stat(path.join(remoteHome, "config.toml"))).mode & 0o777).toBe(0o600); + // A corrupt upload must leave the prior conversation and config intact. + const badTar = path.join(second.runtimeRootDir!, "bad-home-upload.tar"); + await fs.writeFile(badTar, "invalid archive"); + const command = buildCodexHomeRefreshProvision({ mergeAuth }).postUploadCommand!({ + assetDir: remoteHome, assetTarPath: badTar, runtimeRootDir: second.runtimeRootDir!, + }); + const failed = await target.runner.execute({ command: "sh", args: ["-c", command], cwd: remoteCwd }); + expect(failed.exitCode).not.toBe(0); + expect(await fs.readFile(path.join(remoteHome, "sessions", "conversation.jsonl"), "utf8")).toBe("saved-sessions"); + expect(await fs.readFile(path.join(remoteHome, "config.toml"), "utf8")).toContain("https://second.example.test/v1/logs"); + expect((await fs.readdir(remoteHome)).some(name => name.startsWith(".paperclip-home-refresh."))).toBe(false); + } finally { + await Promise.all([...stagedHomes, root].map(dir => fs.rm(dir, { recursive: true, force: true }))); + } + }); + it("stages exactly the allowlist, derefs auth.json to bytes, and excludes decoys", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-stage-")); let staged: string | null = null; diff --git a/packages/adapters/codex-local/src/server/codex-home.ts b/packages/adapters/codex-local/src/server/codex-home.ts index 73ca446738b..a8a83784753 100644 --- a/packages/adapters/codex-local/src/server/codex-home.ts +++ b/packages/adapters/codex-local/src/server/codex-home.ts @@ -2,6 +2,9 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; +import type { SandboxManagedRuntimeAssetProvision } from "@paperclipai/adapter-utils/sandbox-managed-runtime"; +import { shellQuote } from "@paperclipai/adapter-utils/ssh"; +import { buildCodexAuthInboundProvision, CODEX_AUTH_MERGE_EXTRACT_SCRIPT_NAME } from "./codex-auth-merge-scripts.js"; import { resolvePaperclipInstanceRootForAdapter } from "@paperclipai/adapter-utils/server-utils"; import { isCodexAuthCachePath, readSubscriptionAccountId } from "./codex-auth-cache.js"; @@ -27,6 +30,43 @@ export const CODEX_SYNC_ALLOWLIST = [ "skills", ] as const; +/** Refresh only managed entries; remote conversation files never leave their home. + * The staging lease and provider retirement precede this provisioning command. */ +export function buildCodexHomeRefreshProvision(options: { mergeAuth: boolean }): SandboxManagedRuntimeAssetProvision { + const auth = options.mergeAuth ? buildCodexAuthInboundProvision() : undefined; + return { + stageFiles: auth?.stageFiles, + postUploadCommand: ({ assetDir, assetTarPath, runtimeRootDir }) => { + const prepare = options.mergeAuth + ? `if [ -f "$home/auth.json" ]; then cp "$home/auth.json" "$incoming/auth.json"; fi +sh "$runtime_root/${CODEX_AUTH_MERGE_EXTRACT_SCRIPT_NAME}" "$incoming" "$asset_tar"` + : 'tar -xf "$asset_tar" -C "$incoming"\nrm -f "$asset_tar"'; + // Paths arrive confined by the runtime staging layer. Only fixed managed + // names are replaced; sessions, archived sessions and native indexes stay. + const script = `set -eu +umask 077 +home=$1 +asset_tar=$2 +runtime_root=$3 +mkdir -p "$home" +chmod 700 "$home" +incoming=$(mktemp -d "$home/.paperclip-home-refresh.XXXXXX") +trap 'rm -rf -- "$incoming"' EXIT +trap 'exit 1' HUP INT TERM +${prepare} +for name in ${CODEX_SYNC_ALLOWLIST.map(shellQuote).join(" ")}; do + if [ -e "$incoming/$name" ] || [ -L "$incoming/$name" ]; then + if [ -d "$home/$name" ] || [ -L "$home/$name" ]; then rm -rf -- "$home/$name"; fi + mv -f -- "$incoming/$name" "$home/$name" + else + rm -rf -- "$home/$name" + fi +done`; + return `sh -c ${shellQuote(script)} -- ${shellQuote(assetDir)} ${shellQuote(assetTarPath)} ${shellQuote(runtimeRootDir)}`; + }, + }; +} + export type ManagedCodexMcpGateway = { name: string; endpointPath: string; @@ -362,6 +402,7 @@ export async function writeApiKeyAuthJson(home: string, apiKey: string): Promise } export interface StageCodexHomeForSyncOptions { + additionalFiles?: readonly string[]; /** Run id, used only to make the staged temp-dir name traceable in logs. */ runId?: string; } @@ -583,6 +624,13 @@ export async function stageCodexHomeForSync( path.join(os.tmpdir(), `paperclip-codex-home-sync-${runIdPart ? `${runIdPart}-` : ""}`), ); try { + for (const filename of options.additionalFiles ?? []) { + if (!/^paperclip-observability-[A-Za-z0-9_-]+\.config\.toml$/.test(filename)) throw new Error("Invalid managed observability profile"); + const source = path.join(effectiveCodexHome, filename); + const stat = await fs.lstat(source); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (typeof process.getuid === "function" && stat.uid !== process.getuid()) || (stat.mode & 0o077) !== 0) throw new Error("Managed observability profile must be an owned private regular file"); + await fs.writeFile(path.join(stagedHome, filename), await fs.readFile(source), { mode: 0o600, flag: "wx" }); + } for (const entry of CODEX_SYNC_ALLOWLIST) { await stageCodexHomeEntry(effectiveCodexHome, stagedHome, entry); } diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index e81f83f23dd..581f1384f4c 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -1,3 +1,6 @@ +import { asStringArray } from "@paperclipai/adapter-utils/server-utils"; +import { applyObservabilityEnvironment } from "@paperclipai/adapter-utils/agent-observability"; +import { prepareCodexObservabilityProfile, selectedCodexProfile, replaceCodexProfileArgs, disableCodexObservabilityArgs } from "@paperclipai/adapter-utils/codex-observability-profile"; import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary"; import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint"; import { createHash } from "node:crypto"; @@ -777,7 +780,20 @@ export async function execute(ctx: AdapterExecutionContext): Promise> | null = null; + let observabilityPreparationFailed = false; try { + if (ctx.observability) { + if (isManagedCodexHomePath(process.env, agent.companyId, effectiveCodexHome)) { + try { + observabilityProfile = await prepareCodexObservabilityProfile({ home: effectiveCodexHome, runId, + launch: ctx.observability, sourceProfile: selectedCodexProfile(buildCodexExecArgs(config).args) }); + } catch { + observabilityPreparationFailed = true; + await onLog("stderr", "[paperclip] Company observability configuration could not be prepared; execution continues with export disabled.\n"); + } + } else await onLog("stderr", "[paperclip] Company observability is unavailable for a caller-owned Codex home.\n"); + } for (const note of preparedRuntimeConfig.notes) { await onLog("stdout", `[paperclip] ${note}\n`); } @@ -838,7 +854,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise {}, + }); + qualified = version.exitCode === 0 && /^codex-cli 0\.160\.0\s*$/.test(version.stdout.trim()); + } catch { /* Export qualification must not block baseline work. */ } + if (!qualified) { + await observabilityProfile.cleanup(); + observabilityProfile = null; + await onLog("stderr", "[paperclip] Company observability was not applied: this Codex executable is not the qualified 0.160.0 release.\n"); + } + } const loggedEnv = buildInvocationEnvForLogs(env, { runtimeEnv, includeRuntimeKeys: ["HOME"], @@ -1257,6 +1288,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise { }); it("executes Gemini through the shared ACP runtime", async () => { - const root = await makeTempRoot("paperclip-gemini-acp-run-"); + const root = await fs.realpath(await makeTempRoot("paperclip-gemini-acp-run-")); process.env.HOME = path.join(root, "home"); const runtime = new FakeRuntime({}); const metas: AdapterInvocationMeta[] = []; diff --git a/packages/db/src/migrations/0357_naive_wolfpack.sql b/packages/db/src/migrations/0357_naive_wolfpack.sql new file mode 100644 index 00000000000..cacace4e5e0 --- /dev/null +++ b/packages/db/src/migrations/0357_naive_wolfpack.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS "company_observability" ( + "company_id" uuid PRIMARY KEY NOT NULL, + "revision" uuid NOT NULL, + "enabled" boolean DEFAULT false NOT NULL, + "destination" jsonb NOT NULL, + "capture" jsonb NOT NULL, + "credentials_secret_id" uuid, + "header_names" jsonb DEFAULT '[]'::jsonb NOT NULL, + "last_test" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "company_observability" ADD CONSTRAINT "company_observability_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$;--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "company_observability" ADD CONSTRAINT "company_observability_credentials_secret_id_company_secrets_id_fk" FOREIGN KEY ("credentials_secret_id") REFERENCES "public"."company_secrets"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; diff --git a/packages/db/src/migrations/meta/0352_snapshot.json b/packages/db/src/migrations/meta/0357_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0352_snapshot.json rename to packages/db/src/migrations/meta/0357_snapshot.json index d18393a0ece..ada70a59548 100644 --- a/packages/db/src/migrations/meta/0352_snapshot.json +++ b/packages/db/src/migrations/meta/0357_snapshot.json @@ -1,6 +1,6 @@ { - "id": "c13b54d2-e6c0-485b-abf3-5676a739dfe4", - "prevId": "c88ec89d-ff6e-4b78-8b80-068d3375a6d0", + "id": "529b178d-2192-4a6a-8906-2068e5d077af", + "prevId": "043fa318-32fd-4ed6-955f-bf5437ace611", "version": "7", "dialect": "postgresql", "tables": { @@ -15075,6 +15075,110 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_observability": { + "name": "company_observability", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "destination": { + "name": "destination", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "capture": { + "name": "capture", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "credentials_secret_id": { + "name": "credentials_secret_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "header_names": { + "name": "header_names", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_test": { + "name": "last_test", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "company_observability_company_id_companies_id_fk": { + "name": "company_observability_company_id_companies_id_fk", + "tableFrom": "company_observability", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "company_observability_credentials_secret_id_company_secrets_id_fk": { + "name": "company_observability_credentials_secret_id_company_secrets_id_fk", + "tableFrom": "company_observability", + "tableTo": "company_secrets", + "columnsFrom": [ + "credentials_secret_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_onboarding_seeds": { "name": "company_onboarding_seeds", "schema": "", @@ -60844,6 +60948,12 @@ "primaryKey": false, "notNull": true }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "generation": { "name": "generation", "type": "integer", @@ -61032,6 +61142,22 @@ "concurrently": false, "method": "btree", "with": {} + }, + "muse_bindings_active_grant_uq": { + "name": "muse_bindings_active_grant_uq", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"muse_agent_bindings\".\"revoked_at\" IS NULL AND \"muse_agent_bindings\".\"grant_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -61060,6 +61186,19 @@ ], "onDelete": "cascade", "onUpdate": "no action" + }, + "muse_agent_bindings_grant_id_mcp_oauth_grants_id_fk": { + "name": "muse_agent_bindings_grant_id_mcp_oauth_grants_id_fk", + "tableFrom": "muse_agent_bindings", + "tableTo": "mcp_oauth_grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" } }, "compositePrimaryKeys": {}, diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 0cc413c89e4..28a6799dd15 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2486,6 +2486,13 @@ "when": 1792188583719, "tag": "0356_agent_connector_access_hooks", "breakpoints": true + }, + { + "idx": 357, + "version": "7", + "when": 1792439538702, + "tag": "0357_naive_wolfpack", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/company_observability.ts b/packages/db/src/schema/company_observability.ts new file mode 100644 index 00000000000..75331c1c3b4 --- /dev/null +++ b/packages/db/src/schema/company_observability.ts @@ -0,0 +1,17 @@ +import { boolean, jsonb, pgTable, timestamp, uuid } from "drizzle-orm/pg-core"; +import type { AgentObservabilityCapture, AgentObservabilityDestination, AgentObservabilityTestReceipt } from "@paperclipai/shared"; +import { companies } from "./companies.js"; +import { companySecrets } from "./company_secrets.js"; + +export const companyObservability = pgTable("company_observability", { + companyId: uuid("company_id").primaryKey().references(() => companies.id, { onDelete: "cascade" }), + revision: uuid("revision").notNull(), + enabled: boolean("enabled").notNull().default(false), + destination: jsonb("destination").$type().notNull(), + capture: jsonb("capture").$type().notNull(), + credentialsSecretId: uuid("credentials_secret_id").references(() => companySecrets.id, { onDelete: "set null" }), + headerNames: jsonb("header_names").$type().notNull().default([]), + lastTest: jsonb("last_test").$type(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index b12d6b87547..f971f27f267 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -253,3 +253,4 @@ export { executionWorkspaceRepositories } from "./execution_workspace_repositori export { executionWorkspaceRepositoryRoots } from "./execution_workspace_repository_roots.js"; export { executionWorkspaceRepositoryAccess } from "./execution_workspace_repository_access.js"; export * from "./fast_responses.js"; +export { companyObservability } from "./company_observability.js"; diff --git a/packages/paperclip-runner/acpx-profiles.json b/packages/paperclip-runner/acpx-profiles.json index a8881081ca8..b1f2837c5c1 100644 --- a/packages/paperclip-runner/acpx-profiles.json +++ b/packages/paperclip-runner/acpx-profiles.json @@ -14,12 +14,12 @@ "requiresProviderPolicy": false }, "pi": { - "agentProfileVersion": 27, + "agentProfileVersion": 28, "agentServerPackage": "pi-acp", "agentServerVersion": "0.0.33", "agentRuntimePackage": "@earendil-works/pi-coding-agent", "agentRuntimeVersion": "1.0.0", - "commandDigest": "sha256:b1b75a1ae3ac498eb07b95b96bebf4da86227b83d09276076dd5c222e3964dba", + "commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", "requiresProviderPolicy": true }, "cursor": { @@ -32,12 +32,12 @@ "requiresProviderPolicy": true }, "copilot": { - "agentProfileVersion": 23, + "agentProfileVersion": 24, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:6ca1e2c9efa23887ba6f08d89980c91d13114640ac8a1a79f00f27aa8cd39d63", + "commandDigest": "sha256:87e275270e50b42290d10ecec9dfce86ffca2b7a7519cb60c584004adfc86437", "requiresProviderPolicy": true, "qualificationStatus": "pending" }, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 6537fbbfeb8..81a535da873 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -3200,7 +3200,7 @@ mod tests { #[test] fn pi_profile_matches_published_identity_and_rejects_prior_profiles() { let published: Value = serde_json::from_str(include_str!( - "../../../../test-fixtures/pi-acp/profile-v27-identity.json" + "../../../../test-fixtures/pi-acp/profile-v28-identity.json" )) .unwrap(); let mut value = descriptor("codex"); @@ -3234,6 +3234,7 @@ mod tests { ), include_str!("../../../../test-fixtures/pi-acp/profile-v25-identity.json"), include_str!("../../../../test-fixtures/pi-acp/profile-v26-identity.json"), + include_str!("../../../../test-fixtures/pi-acp/profile-v27-identity.json"), ] { let prior: Value = serde_json::from_str(prior).unwrap(); value["commandDigest"] = prior["commandDigest"].clone(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 830ef41cfd7..fe6d3a84f13 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -258,6 +258,33 @@ impl AcpxSidecarTransport { if agent == "claude" { keys.extend_from_slice(&[ "ANTHROPIC_BASE_URL", + "CLAUDE_CODE_ENABLE_TELEMETRY", + "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", + "ENABLE_ENHANCED_TELEMETRY_BETA", + "OTEL_LOGS_EXPORTER", + "OTEL_TRACES_EXPORTER", + "OTEL_METRICS_EXPORTER", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "OTEL_LOG_USER_PROMPTS", + "OTEL_LOG_ASSISTANT_RESPONSES", + "OTEL_LOG_TOOL_DETAILS", + "OTEL_LOG_TOOL_CONTENT", + "OTEL_LOG_RAW_API_BODIES", "CLAUDE_CODE_USE_BEDROCK", "AWS_REGION", "AWS_DEFAULT_REGION", @@ -270,6 +297,13 @@ impl AcpxSidecarTransport { "CLAUDE_CODE_SUBAGENT_MODEL", ]); } + if agent == "codex" { + keys.extend_from_slice(&[ + "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + ]); + } let pi_keys = if agent == "pi" { pi_credential_environment_keys( std::env::var("PAPERCLIP_ACPX_CREDENTIAL_BINDING") diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 67e4ae38399..b6f7675c5e0 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -821,6 +821,8 @@ const GITHUB_CREDENTIAL_ENVIRONMENT_KEYS: &[&str] = &[ ]; const CODEX_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[ + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "PAPERCLIP_AI_PROVIDER_KEY", "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/configured_environment.rs b/packages/paperclip-runner/runner/crates/runner-core/src/configured_environment.rs index 2fed99a63e0..aba51dbbcde 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/configured_environment.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/configured_environment.rs @@ -4,6 +4,34 @@ use std::process::Command; const MARKER: &str = "PAPERCLIP_CONFIGURED_ENV_KEYS"; const RESERVED: &[&str] = &[ + "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG", + "CLAUDE_CODE_ENABLE_TELEMETRY", + "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", + "ENABLE_ENHANCED_TELEMETRY_BETA", + "OTEL_LOGS_EXPORTER", + "OTEL_TRACES_EXPORTER", + "OTEL_METRICS_EXPORTER", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "OTEL_LOG_USER_PROMPTS", + "OTEL_LOG_ASSISTANT_RESPONSES", + "OTEL_LOG_TOOL_DETAILS", + "OTEL_LOG_TOOL_CONTENT", + "OTEL_LOG_RAW_API_BODIES", "AGENT_HOME", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs index 7ad167ce577..9fc6e08fbc9 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs @@ -29,7 +29,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: Some("@earendil-works/pi-coding-agent"), agent_runtime_version: Some("1.0.0"), command_digest: - "sha256:b1b75a1ae3ac498eb07b95b96bebf4da86227b83d09276076dd5c222e3964dba", + "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", requires_provider_policy: true, }, "cursor" => AcpxReleaseProfile { @@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: None, agent_runtime_version: None, command_digest: - "sha256:6ca1e2c9efa23887ba6f08d89980c91d13114640ac8a1a79f00f27aa8cd39d63", + "sha256:87e275270e50b42290d10ecec9dfce86ffca2b7a7519cb60c584004adfc86437", requires_provider_policy: true, }, "claude" => AcpxReleaseProfile { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 4198fbb33a4..0ed2cf97667 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -17,7 +17,7 @@ use sha2::{Digest, Sha256}; const CODEX_ACPX_DIGEST: &str = "sha256:443491f4e814501a5dc47347e4510e84faeec9c9382e2d4e53bfe69386d5aabc"; const PI_ACPX_DIGEST: &str = - "sha256:b1b75a1ae3ac498eb07b95b96bebf4da86227b83d09276076dd5c222e3964dba"; + "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c"; fn temporary_directory(label: &str) -> PathBuf { let nonce = SystemTime::now() diff --git a/packages/paperclip-runner/src/backends/codex-native-backend.ts b/packages/paperclip-runner/src/backends/codex-native-backend.ts index 36a4eb298ca..56b7ac0966d 100644 --- a/packages/paperclip-runner/src/backends/codex-native-backend.ts +++ b/packages/paperclip-runner/src/backends/codex-native-backend.ts @@ -23,6 +23,7 @@ import { export interface CodexNativeSessionBackendOptions { /** Effective provider environment, including the assigned workspace boundary. */ environment?: NodeJS.ProcessEnv; + observability?: import("../drivers/agent-observability.js").ProviderObservabilityLaunch; /** Filesystem that authoritatively admits the workspace path. */ workingDirectoryAuthority?: CodexWorkingDirectoryAuthority; runnerInstanceId?: string; @@ -191,6 +192,7 @@ function createTransportBackedNativeSessionBackend( dynamicToolHandler: options.dynamicToolHandler, completionFeedback: options.completionFeedback, environment: options.environment, + observability: options.observability, workingDirectoryAuthority: options.workingDirectoryAuthority, driverIdentity, capabilities: isCodex diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.ts b/packages/paperclip-runner/src/backends/native-backend-factory.ts index 18e17957202..f50bd0c9a4f 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.ts @@ -74,6 +74,7 @@ export function createNativeSessionBackend( dynamicTools: options.dynamicTools, dynamicToolHandler: options.dynamicToolHandler, environment: options.environment, + observability: options.observability, workingDirectoryAuthority: options.workingDirectoryAuthority, transportFactory: options.codexTransportFactory, }); @@ -126,6 +127,7 @@ export function createNativeSessionBackend( dynamicTools: options.dynamicTools, dynamicToolHandler: options.dynamicToolHandler, environment: options.environment, + observability: options.observability, workingDirectoryAuthority: options.workingDirectoryAuthority, transportFactory: options.codexTransportFactory, }); diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index 08007314eb0..a032ac93e35 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -1,4 +1,5 @@ import { acpxProfileActivity } from "../drivers/acpx/profile-activity.js"; +import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js"; import { appendSemanticToolReceipt, readNativeSemanticReceipt, semanticInputSha256 } from "../drivers/semantic-tool-receipt.js"; import { startRunnerToolBridge, type RunnerToolCall } from "../drivers/runner-tool-bridge.js"; import { validatePrpStructuredRunResult } from "../protocol/replay-contract.js"; @@ -85,6 +86,7 @@ import { const children = new Set(); afterEach(async () => { + vi.unstubAllEnvs(); await Promise.all([...children].map((child) => child.close())); children.clear(); }); @@ -433,6 +435,8 @@ describe("qualified ACPX runtime sidecar", () => { it.each(["success", "accepted-false", "error", "cancel", "timeout", "receiver-rejected"])( "commits normalized delivery only after the actual tool.resolve handler succeeds: %s", async mode => { + const credentialCanary = "custom-observability-credential-canary"; + if (mode === "error") vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", `x-custom=${credentialCanary}`); const raw = { reportedWorkDisposition: "done", summary: "Complete", evidence: [{ ref: mode === "receiver-rejected" ? "x".repeat(300 * 1024) : "fixture" }], verification: [], completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] } }; const validation = validatePrpStructuredRunResult(raw); @@ -461,7 +465,7 @@ describe("qualified ACPX runtime sidecar", () => { if (mode === "success" || mode === "accepted-false") { await expect(commands.resolve(params)).resolves.toEqual({ resolved: true }); } else if (mode === "error") { - await expect(commands.resolve({ ...params, result: undefined, error: { message: "Receiver rejected completion" } })).resolves.toEqual({ resolved: true }); + await expect(commands.resolve({ ...params, result: undefined, error: { message: `Receiver rejected completion: ${credentialCanary}` } })).resolves.toEqual({ resolved: true }); } else if (mode === "cancel") { const cancelled = await fetch(bridge.url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: 3 } }) }); @@ -473,6 +477,10 @@ describe("qualified ACPX runtime sidecar", () => { expect(Buffer.byteLength(wire.writes[0]!)).toBeLessThan(ACPX_SIDECAR_MAX_FRAME_BYTES); } const body = await (await response).json(); + if (mode === "error") { + expect(JSON.stringify(body)).not.toContain(credentialCanary); + expect(JSON.stringify(body)).toContain("[REDACTED]"); + } const delivered = mode === "success" || mode === "accepted-false"; const receipt = readNativeSemanticReceipt({ contents: body.result.content }); expect(receipt).toMatchObject({ normalizedInputSha256: delivered ? semanticInputSha256(validation.result) : null, @@ -1390,7 +1398,7 @@ function loadActualToolCommands(tools: Map): { const start = source.indexOf("async function dispatch("), end = source.indexOf("\nasync function pumpTurn(", start); const safeStart = source.indexOf("function safeText("), safeEnd = source.indexOf("\nfunction safeMessage(", safeStart); if (start < 0 || end < 0 || safeStart < 0 || safeEnd < 0) throw new Error("Actual sidecar command dispatcher not found"); - const dispatcher = new Function("tools", "boundedIdentity", "record", "text", "parseAcpxSidecarRequest", "ACPX_SIDECAR_PROTOCOL_VERSION", ` + const dispatcher = new Function("tools", "boundedIdentity", "record", "text", "parseAcpxSidecarRequest", "ACPX_SIDECAR_PROTOCOL_VERSION", "redactObservabilityDiagnostic", ` let turnId="test-turn", requestId=0; ${stripTypeScriptTypes(source.slice(safeStart, safeEnd))} ${stripTypeScriptTypes(source.slice(start, end))} @@ -1398,6 +1406,6 @@ function loadActualToolCommands(tools: Map): { resolve: params => dispatch(parseAcpxSidecarRequest({ protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, id: ++requestId, command: "tool.resolve", params })), setTurn: value => { turnId=value; } }; - `)(tools, boundedIdentity, record, text, parseAcpxSidecarRequest, ACPX_SIDECAR_PROTOCOL_VERSION); + `)(tools, boundedIdentity, record, text, parseAcpxSidecarRequest, ACPX_SIDECAR_PROTOCOL_VERSION, redactObservabilityDiagnostic); return dispatcher; } diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index 64a86ef1c39..1789079a809 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js"; import { resolveClaudeEffort } from "../drivers/acpx/claude-effort.js"; import { parseProviderMode } from "../contracts/provider-mode.js"; import { acpxProfileActivity, type AcpxActivityAdapter, type AcpxToolEvidence } from "../drivers/acpx/profile-activity.js"; @@ -1531,7 +1532,7 @@ function safeText(value: unknown, maxBytes = 8_192): string { } function redactSecrets(value: string): string { - return value.replace( + return redactObservabilityDiagnostic(value, process.env).replace( /(key|token|secret|password|authorization)\s*[:=]\s*[^\s,}\]]+/gi, "$1=[REDACTED]", ); diff --git a/packages/paperclip-runner/src/configured-environment.test.ts b/packages/paperclip-runner/src/configured-environment.test.ts index 2281becd0a7..77589d9d40b 100644 --- a/packages/paperclip-runner/src/configured-environment.test.ts +++ b/packages/paperclip-runner/src/configured-environment.test.ts @@ -27,10 +27,11 @@ describe('controller-configured environment across native runner boundaries', () }); it('passes configured Pages values through the runner spawn boundary', () => { let launched: NodeJS.ProcessEnv | undefined; - spawnRunner({ connection: { mode: 'connect', connectUrl: 'ws://127.0.0.1:43127' }, stateDirectory: '/tmp/runner-env-proof', identity, ticket: 'fake-ticket', maxOutboxBytes: 262144, p0ReserveBytes: 65536, runnerVersion: '0.3.0', runnerDigest: `sha256:${'a'.repeat(64)}`, environment: source, + spawnRunner({ connection: { mode: 'connect', connectUrl: 'ws://127.0.0.1:43127' }, stateDirectory: '/tmp/runner-env-proof', identity, ticket: 'fake-ticket', maxOutboxBytes: 262144, p0ReserveBytes: 65536, runnerVersion: '0.3.0', runnerDigest: `sha256:${'a'.repeat(64)}`, environment: { ...source, OTEL_EXPORTER_OTLP_LOGS_HEADERS: 'authorization=Bearer%20export-canary', OTEL_LOGS_EXPORTER: 'otlp' }, processLauncher: spec => { launched = spec.environment; return { child: { pid: 42, exitCode: null, signalCode: null, kill: () => true }, completion: Promise.resolve({ code: 0, signal: null, stdout: '', stderr: '' }) }; }, }); - expect(launched).toMatchObject(pages); + expect(launched).toMatchObject({ ...pages, OTEL_EXPORTER_OTLP_LOGS_HEADERS: 'authorization=Bearer%20export-canary', OTEL_LOGS_EXPORTER: 'otlp' }); + expect(launched!.PAPERCLIP_CONFIGURED_ENV_KEYS).not.toContain('OTEL_'); expect(launched!.DATABASE_URL).toBeUndefined(); expect(launched!.NODE_OPTIONS).toBeUndefined(); expect(launched!.PAPERCLIP_API_KEY).toBeUndefined(); diff --git a/packages/paperclip-runner/src/configured-environment.ts b/packages/paperclip-runner/src/configured-environment.ts index 49a9be538f3..3cca642989f 100644 --- a/packages/paperclip-runner/src/configured-environment.ts +++ b/packages/paperclip-runner/src/configured-environment.ts @@ -1,3 +1,4 @@ +import { AGENT_OBSERVABILITY_ENV_KEYS } from "./drivers/agent-observability.js"; /** Names minted by the controller from the resolved task configuration. * Values remain in the process environment, never in argv or this manifest. * Do not mint this projection from ambient process.env. @@ -6,6 +7,7 @@ export const CONFIGURED_ENVIRONMENT_KEYS = "PAPERCLIP_CONFIGURED_ENV_KEYS"; export const GENERATED_RUNTIME_ENVIRONMENT_KEYS: ReadonlySet = new Set([ CONFIGURED_ENVIRONMENT_KEYS, + ...AGENT_OBSERVABILITY_ENV_KEYS, "PAPERCLIP_AGENT_ID", "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY", "PAPERCLIP_API_KEY", "PAPERCLIP_API_URL", "PAPERCLIP_API_BRIDGE_MODE", "PAPERCLIP_COMPANY_ID", "PAPERCLIP_INSTANCE_ID", "PAPERCLIP_RUN_ID", "PAPERCLIP_NORMALIZED_SESSION_ID", "PAPERCLIP_RUNNER_INSTANCE_ID", "PAPERCLIP_RUNNER_BOOTSTRAP_TICKET", "PAPERCLIP_TASK_ID", "PAPERCLIP_APPROVAL_ID", "PAPERCLIP_APPROVAL_STATUS", "PAPERCLIP_LINKED_ISSUE_IDS", diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index d86aafdca7e..b9b8c35bd6e 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -1,3 +1,4 @@ +import { AGENT_OBSERVABILITY_ENV_KEYS } from "../drivers/agent-observability.js"; import { configuredEnvironment } from "../configured-environment.js"; import { spawn } from "node:child_process"; import { @@ -3479,7 +3480,7 @@ function runnerEnvironment( // already-sanitized environment for this run. Never inherit them implicitly // from the server process. if (explicitSource !== undefined) { - for (const key of runnerExplicitProviderEnvironmentKeys) { + for (const key of [...runnerExplicitProviderEnvironmentKeys, ...AGENT_OBSERVABILITY_ENV_KEYS]) { const value = explicitSource[key]; if (value !== undefined) environment[key] = value; } diff --git a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts index 12ba72ccf45..00e9660fa55 100644 --- a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts @@ -225,8 +225,18 @@ describe("Copilot build-owned installation", () => { expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBe("pending"); }); - it("admits the current v23 declaration and binds its source policy, receipts and patch hashes", () => { - const identity = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v23-identity.json", import.meta.url), "utf8")); + it("preserves v23 history while binding shared observability diagnostic redaction", () => { + const prior = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v23-identity.json", import.meta.url), "utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v24-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "semanticSidecarSourceSha256", + ]); + expect(isSupportedAcpxProfileVersion("copilot", 23)).toBe(true); + expect(current.commandDigest).not.toBe(prior.commandDigest); + }); + + it("admits the current v24 declaration and binds its source policy, receipts and patch hashes", () => { + const identity = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v24-identity.json", import.meta.url), "utf8")); expect(identity.declaration.systemInstructionDelivery).toBe(COPILOT_SYSTEM_INSTRUCTION_DELIVERY); expect(identity.declaration.sharedRuntimeContract).toBe("paperclip.acpx-runtime-contract.v1"); expect(identity.declaration.permissionContextContract).toBe(COPILOT_PERMISSION_CONTEXT_CONTRACT); @@ -257,7 +267,7 @@ describe("Copilot build-owned installation", () => { expect([...closure].filter(path => !bound.has(path))).toEqual(["negative-control:unbound-policy"]); }); it.each([...PERMISSION_POLICY_SOURCES, ...SEMANTIC_RECEIPT_SOURCES])("changing %s changes the candidate identity", (_path, field) => { - const identity = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v23-identity.json", import.meta.url), "utf8")); + const identity = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v24-identity.json", import.meta.url), "utf8")); identity.declaration[field] = "0".repeat(64); const sorted = Object.fromEntries(Object.entries(identity.declaration).sort(([a], [b]) => a.localeCompare(b))); expect(`sha256:${createHash("sha256").update(JSON.stringify(sorted)).digest("hex")}`).not.toBe(identity.commandDigest); diff --git a/packages/paperclip-runner/src/drivers/acpx/environment.ts b/packages/paperclip-runner/src/drivers/acpx/environment.ts index bab0d645904..1fb79afba46 100644 --- a/packages/paperclip-runner/src/drivers/acpx/environment.ts +++ b/packages/paperclip-runner/src/drivers/acpx/environment.ts @@ -1,4 +1,5 @@ import { isOpenRouterUrl, openRouterClaudeHeaders } from "../openrouter-attribution.js"; +import { providerObservabilityEnvironment } from "../agent-observability.js"; import { CONFIGURED_ENVIRONMENT_KEYS, configuredEnvironment, configuredEnvironmentKeys } from "../../configured-environment.js"; import { PI_CREDENTIAL_NAMES, PI_GENERAL_IAM_CREDENTIAL_NAMES, piCredentialNames } from "./pi-provider-config.js"; import { PRECONFIGURED_HOME_KEYS, usesPreconfiguredHome } from "../preconfigured-home.js"; @@ -103,7 +104,7 @@ export function createSanitizedAcpxSpawnInput( ): SanitizedAcpxSpawnInput { const source = environment ?? process.env; const candidate = isCandidate(agent); - const result: NodeJS.ProcessEnv = configuredEnvironment(environment); + const result: NodeJS.ProcessEnv = { ...configuredEnvironment(environment), ...providerObservabilityEnvironment(environment, agent) }; const credentialNames = agent === "pi" ? piCredentialNames(environment) : ACPX_CREDENTIAL_NAMES[agent]; const allowed = new Set([ "PATH", diff --git a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts index bf7d198f0b9..7e479b476bb 100644 --- a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts +++ b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts @@ -22,12 +22,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "pi", - "agentProfileVersion": 27, + "agentProfileVersion": 28, "agentServerPackage": "pi-acp", "agentServerVersion": "0.0.33", "agentRuntimePackage": "@earendil-works/pi-coding-agent", "agentRuntimeVersion": "1.0.0", - "commandDigest": "sha256:b1b75a1ae3ac498eb07b95b96bebf4da86227b83d09276076dd5c222e3964dba", + "commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", "permissionPolicy": "interactive" }, "cursor": { @@ -48,12 +48,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "copilot", - "agentProfileVersion": 23, + "agentProfileVersion": 24, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:6ca1e2c9efa23887ba6f08d89980c91d13114640ac8a1a79f00f27aa8cd39d63", + "commandDigest": "sha256:87e275270e50b42290d10ecec9dfce86ffca2b7a7519cb60c584004adfc86437", "qualificationStatus": "pending", "permissionPolicy": "interactive" }, diff --git a/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts index 220e20c9e72..7695e64219c 100644 --- a/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts @@ -147,11 +147,19 @@ describe("Pi installation factory", () => { } }); + it("preserves profile 27 and binds shared observability handling without changing Pi runtime bytes", async () => { + const read = async (version: number) => JSON.parse(await readFile(new URL(`../../../test-fixtures/pi-acp/profile-v${version}-identity.json`, import.meta.url), "utf8")); + const prior = await read(27), current = await read(28); + expect(Object.keys(current.declaration).filter(key => JSON.stringify(current.declaration[key]) !== JSON.stringify(prior.declaration[key])).sort()) + .toEqual(["agentProfileVersion", "credentialEnvironmentSourceSha256", "runtimeSandboxSourceSha256"]); + expect(current.commandDigest).not.toBe(prior.commandDigest); + }); + it("binds the profile declaration to the reviewed patch and platform closure pins", async () => { const hash = (bytes: string | Uint8Array) => createHash("sha256").update(bytes).digest("hex"); const canonical = (value: any): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}` : JSON.stringify(value); - const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v27-identity.json", import.meta.url), "utf8")); + const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v28-identity.json", import.meta.url), "utf8")); const declaration = identity.declaration; expect(declaration.sharedRuntimeContract).toBe("paperclip.acpx-runtime-contract.v1"); expect(identity.commandDigest).toBe(`sha256:${hash(canonical(declaration))}`); @@ -182,21 +190,21 @@ describe("Pi installation factory", () => { it("rejects legacy profiles and tampered executable identities", () => { expect(() => assertPiInstallationProfile(candidate())).not.toThrow(); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 27"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 27"); - for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 27"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 28"); + for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 28"); for (const changed of [{ agentServerVersion: "latest" }, { commandDigest: `sha256:${"0".repeat(64)}` }, { agentRuntimePackage: "ambient-pi" }]) { expect(() => assertPiInstallationProfile({ ...candidate(), ...changed })).toThrow("trusted declaration"); } diff --git a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts index 213dd87a81d..b2f3ad7066e 100644 --- a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts +++ b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts @@ -2,13 +2,13 @@ import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js"; /** Decodable historical profile revisions, not permission to launch them. * Exact current profile/package/digest admission is checked separately. */ -type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26; +type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27; type CurrentAcpxProfileVersion = typeof QUALIFIED_ACPX_PROFILE_DATA[keyof typeof QUALIFIED_ACPX_PROFILE_DATA]["agentProfileVersion"]; export type AcpxProfileVersion = HistoricalAcpxProfileVersion | CurrentAcpxProfileVersion; const historicalVersions: Readonly> = { - pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], - grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22], + pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], + grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], cursor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], }; diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts index 6c4b9ce4833..3b32e306121 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts @@ -42,6 +42,8 @@ describe("ACPX runtime sandbox", () => { const pages = { PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY: "task-pages-secret", CUSTOM_TOKEN: "task-service-secret" }; const sandbox = await prepareAcpxRuntimeSandbox({ binding: fixture.binding, agent: "codex", environment: { PATH: process.env.PATH, OPENAI_API_KEY: "provider-secret", HOST_SECRET: "unselected-secret", + PAPERCLIP_OBSERVABILITY_CODEX_CONFIG: JSON.stringify({ exporter: "none", trace_exporter: "none", metrics_exporter: "none" }), + OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20export-canary", ...configuredEnvironmentProjection(pages), ...(identity ? { PAPERCLIP_AGENT_KEY_ID: "agent-key" } : {}), } }); const config = await readFile(join(sandbox.agentHomeDirectory, "config.toml"), "utf8"); @@ -49,6 +51,9 @@ describe("ACPX runtime sandbox", () => { expect(keys).toEqual(expect.arrayContaining(Object.keys(pages))); expect(keys).not.toContain("OPENAI_API_KEY"); expect(keys).not.toContain("HOST_SECRET"); + expect(keys).not.toContain("PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"); + expect(keys).not.toContain("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + expect(sandbox.persistedEnvironment).not.toHaveProperty("PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"); expect(sandbox.launchEnvironment).toMatchObject(pages); for (const value of Object.values(pages)) expect(config).not.toContain(value); }); diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts index 8f99b671010..0a2f679bbaf 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts @@ -1,3 +1,4 @@ +import { AGENT_OBSERVABILITY_ENV_KEYS, codexObservabilityToml } from "../agent-observability.js"; import { piProviderConfiguration } from "./pi-provider-config.js"; import { configuredEnvironmentKeys } from "../../configured-environment.js"; import { COPILOT_SYSTEM_INSTRUCTIONS_FILE } from "./copilot-profile.js"; @@ -489,6 +490,7 @@ export async function prepareAcpxRuntimeSandbox(input: { // durable runtime state. Keep this identical to the proven native // Codex isolation policy: broader shell-environment filtering can // also affect provider startup and belongs at the launch boundary. + ...(input.environment?.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG ? [codexObservabilityToml(JSON.parse(input.environment.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG))] : []), "[features]", "shell_snapshot = false", ...(input.environment?.PAPERCLIP_AGENT_KEY_ID || taskEnvironmentKeys.length > 0 ? [ @@ -499,7 +501,7 @@ export async function prepareAcpxRuntimeSandbox(input: { ...taskEnvironmentKeys, // Unselected provider/config secrets keep Codex's default shell exclusions; // only Paperclip's scoped API token is required by Bash/curl skills. - ...Object.keys(input.environment ?? {}).filter(key => key === "PAPERCLIP_API_KEY" || !/key|secret|token/i.test(key)), + ...Object.keys(input.environment ?? {}).filter(key => !(AGENT_OBSERVABILITY_ENV_KEYS as readonly string[]).includes(key) && (key === "PAPERCLIP_API_KEY" || !/key|secret|token/i.test(key))), ])])}`, ] : []), "", diff --git a/packages/paperclip-runner/src/drivers/agent-observability.test.ts b/packages/paperclip-runner/src/drivers/agent-observability.test.ts new file mode 100644 index 00000000000..4fdc2f93348 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/agent-observability.test.ts @@ -0,0 +1,52 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse } from "smol-toml"; +import { prepareIsolatedCodexHome } from "./runtime-context-materializer.js"; +import { AGENT_OBSERVABILITY_ENV_KEYS, applyProviderObservabilityEnvironment, providerObservabilityEnvironment, redactObservabilityDiagnostic, type ProviderObservabilityLaunch } from "./agent-observability.js"; +import { createSanitizedAcpxSpawnInput } from "./acpx/environment.js"; +import { configuredEnvironmentProjection } from "../configured-environment.js"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000" }, otelConfig: { + exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: { authorization: "Bearer secret-canary" } } }, metrics_exporter: "none", log_agent_responses: true, +} }; +it("transports a valid encoded header set in both Codex exporters without rejecting the launch", () => { + const headers = Object.fromEntries([0, 1, 2, 3].map(index => [`x-custom-${index}`, encodeURIComponent("%".repeat(4000))])); + const exporter = { "otlp-http": { headers } }; + const value = JSON.stringify({ exporter, trace_exporter: exporter }); + expect(Buffer.byteLength(value)).toBeGreaterThan(65536); + expect(providerObservabilityEnvironment({ PAPERCLIP_OBSERVABILITY_CODEX_CONFIG: value }, "codex")).toEqual({ PAPERCLIP_OBSERVABILITY_CODEX_CONFIG: value }); + expect(() => providerObservabilityEnvironment({ PAPERCLIP_OBSERVABILITY_CODEX_CONFIG: "x".repeat(128 * 1024 + 1) }, "codex")).toThrow("Invalid observability launch environment"); +}); +it("materializes the real native home with both governed MCP and protected export config", async () => { + const home = await mkdtemp(join(tmpdir(), "paperclip-native-otel-")); roots.push(home); + await mkdir(join(home, "source")); + await writeFile(join(home, "source", "auth.json"), '{"OPENAI_API_KEY":"fixture"}'); + await prepareIsolatedCodexHome({ context: null, codexHome: home, sourceCodexHome: join(home, "source"), observability: launch, + nativeMcp: { name: "paperclip-assigned", url: "https://paperclip.example/mcp", token: "mcp-secret" } }); + const config = parse(await readFile(join(home, "config.toml"), "utf8")); + expect(config.otel).toEqual(launch.otelConfig); expect(config.mcp_servers).toHaveProperty("paperclip-assigned"); + expect((await stat(join(home, "config.toml"))).mode & 0o777).toBe(0o600); +}); +it("passes only explicit Claude observability through its sanitizer and excludes task projection", () => { + const source = { OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary", KEEP: "task" }; + expect(createSanitizedAcpxSpawnInput(source, "claude").env).toMatchObject({ OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: source.OTEL_EXPORTER_OTLP_LOGS_HEADERS }); + expect(createSanitizedAcpxSpawnInput(source, "pi").env).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + const projected = configuredEnvironmentProjection(source); + expect(projected.PAPERCLIP_CONFIGURED_ENV_KEYS).toBe('["KEEP"]'); + expect(redactObservabilityDiagnostic("export authorization=Bearer secret-canary", source)).not.toContain("secret-canary"); +}); +it("clears stale launch values and serializes Codex only for its trusted ACPX process", () => { + const environment = applyProviderObservabilityEnvironment({ OTEL_EXPORTER_OTLP_HEADERS: "stale" }, launch, true); + expect(environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_HEADERS"); + expect(environment.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG).toBe(JSON.stringify(launch.otelConfig)); + expect(AGENT_OBSERVABILITY_ENV_KEYS).toContain("PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"); +}); +it("redacts encoded Codex header values and their decoded transport form", () => { + const raw = "Bearer arbitrary%26secret,with=punctuation"; + const config = { exporter: { "otlp-http": { headers: { authorization: encodeURIComponent(raw) } } } }; + const diagnostic = redactObservabilityDiagnostic(`${raw} ${raw.slice(7)} ${encodeURIComponent(raw)}`, {}, config); + expect(diagnostic).toBe("[REDACTED] [REDACTED] [REDACTED]"); +}); diff --git a/packages/paperclip-runner/src/drivers/agent-observability.ts b/packages/paperclip-runner/src/drivers/agent-observability.ts new file mode 100644 index 00000000000..37b11a13032 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/agent-observability.ts @@ -0,0 +1,92 @@ +import { parse, stringify } from "smol-toml"; +import { chmodSync, lstatSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; + +/** Controller-owned, secret-bearing options; excluded from all durable descriptors. */ +export interface ProviderObservabilityLaunch { + kind: "codex" | "claude"; + compatibilityKey: string; + otelConfig?: Record; + environment: Readonly>; +} +export const AGENT_OBSERVABILITY_ENV_KEYS = [ + "CLAUDE_CODE_ENABLE_TELEMETRY", "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "ENABLE_ENHANCED_TELEMETRY_BETA", + "OTEL_LOGS_EXPORTER", "OTEL_TRACES_EXPORTER", "OTEL_METRICS_EXPORTER", "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT", "OTEL_LOG_RAW_API_BODIES", + "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG", +] as const; + +export function providerObservabilityEnvironment(source: NodeJS.ProcessEnv | undefined, agent: string): NodeJS.ProcessEnv { + if (!source || !["codex", "claude"].includes(agent)) return {}; + const result: NodeJS.ProcessEnv = {}; + const names: readonly string[] = agent === "claude" ? AGENT_OBSERVABILITY_ENV_KEYS.filter(key => key !== "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG") + : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"]; + for (const key of names) if (source[key] !== undefined) { + // Codex carries two percent-encoded header maps. A valid 16-KiB company + // header set can exceed 64 KiB after encoding and duplication. + const maxBytes = key === "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG" ? 128 * 1024 : 65536; + if (source[key]!.includes("\0") || Buffer.byteLength(source[key]!) > maxBytes) throw new Error("Invalid observability launch environment"); + result[key] = source[key]; + } + return result; +} + +export function applyProviderObservabilityEnvironment(environment: NodeJS.ProcessEnv, launch: ProviderObservabilityLaunch | undefined, acpx = false): NodeJS.ProcessEnv { + if (!launch) return environment; + const result = { ...environment }; + for (const key of AGENT_OBSERVABILITY_ENV_KEYS) delete result[key]; + Object.assign(result, launch.environment); + if (acpx && launch.kind === "codex" && launch.otelConfig) result.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG = JSON.stringify(launch.otelConfig); + return result; +} + +export function codexObservabilityToml(config: Record | undefined): string { + return config ? stringify({ otel: config }) : ""; +} + +/** Only called with a controller-owned provider home, never a discovered user home. */ +export function writeManagedCodexObservability(home: string, config: Record): void { + const path = join(home, "config.toml"); + const stat = lstatSync(path, { throwIfNoEntry: false }); + if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw new Error("Invalid managed Codex config"); + const current = stat ? parse(readFileSync(path, "utf8")) : {}; + const temporary = join(home, `.config-${randomUUID()}.tmp`); + try { + writeFileSync(temporary, stringify({ ...current, otel: config }), { mode: 0o600, flag: "wx" }); + renameSync(temporary, path); + chmodSync(path, 0o600); + } finally { rmSync(temporary, { force: true }); } +} + +/** Exact values, including encoded header bundles, must be removed before diagnostics persist. */ +export function redactObservabilityDiagnostic(message: string, environment: NodeJS.ProcessEnv | undefined, config?: Record): string { + const values = new Set(); + for (const key of ["OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"]) { + const value = environment?.[key]; + if (!value) continue; + values.add(value); + for (const item of value.split(",")) { + const encoded = item.slice(item.indexOf("=") + 1); + try { values.add(decodeURIComponent(encoded)); } catch { values.add(encoded); } + } + } + const codex = config ?? (() => { try { return JSON.parse(environment?.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG ?? "null"); } catch { return null; } })(); + for (const exporter of [codex?.exporter, codex?.trace_exporter]) { + const headers = (exporter as { "otlp-http"?: { headers?: Record } } | undefined)?.["otlp-http"]?.headers; + for (const value of Object.values(headers ?? {})) { + values.add(value); + try { values.add(decodeURIComponent(value)); } catch { /* Older launch contracts may contain literal percent values. */ } + } + } + for (const value of [...values]) if (/^Bearer /i.test(value)) values.add(value.slice(7)); + for (const value of [...values]) values.add(JSON.stringify(value).slice(1, -1)); + for (const value of [...values].filter(Boolean).sort((a, b) => b.length - a.length)) message = message.replaceAll(value, "[REDACTED]"); + return message; +} diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index 548126ad4d2..32243ffaf7a 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -204,6 +204,7 @@ class BoundedLineDecoder { } const SAFE_ENVIRONMENT_KEYS = [ + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "AGENT_HOME", "ALL_PROXY", "CODEX_HOME", diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index f340432a9c4..cf32d893611 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -1,4 +1,5 @@ import { usesPreconfiguredHome } from "../preconfigured-home.js"; +import { writeManagedCodexObservability, redactObservabilityDiagnostic } from "../agent-observability.js"; import { trustCodexStartupRoot } from "./codex-startup-trust.js"; import { readCodexThreadState, readCodexTurnMetadata } from "./codex-history.js"; import { codexExecutableReadOnlyRoots } from "./codex-security-config.js"; @@ -671,6 +672,11 @@ export class CodexAppServerDriver implements HarnessDriver { >; }): CodexAppServerTransport { const workingDirectory = context?.workingDirectory ?? this.#options.environment?.PAPERCLIP_WORKSPACE_CWD; + if (this.#options.observability?.otelConfig && !this.#options.transportFactory && !usesPreconfiguredHome(this.#options.environment)) { + const home = this.#options.environment?.CODEX_HOME; + if (home) writeManagedCodexObservability(home, this.#options.observability.otelConfig); + else this.#options.onDiagnostic?.("Company observability requires a managed Codex home"); + } if (!usesPreconfiguredHome(this.#options.environment) && !this.#options.transportFactory && this.#options.environment?.CODEX_HOME && workingDirectory) { trustCodexStartupRoot(this.#options.environment.CODEX_HOME, workingDirectory); } @@ -680,7 +686,7 @@ export class CodexAppServerDriver implements HarnessDriver { workingDirectory, args: createIsolatedCodexAppServerArgs(this.#options.environment, codexExecutableReadOnlyRoots(this.#options.environment ?? process.env), this.#options.instructionWorkingCopyRoot), environment: createSanitizedCodexEnvironment(this.#options.environment), - onDiagnostic: this.#options.onDiagnostic, + onDiagnostic: message => this.#options.onDiagnostic?.(redactObservabilityDiagnostic(message, this.#options.environment, this.#options.observability?.otelConfig)), processGroup: true, }) ); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts index ef2b1864d42..a9256351aa5 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts @@ -59,6 +59,7 @@ export interface CodexAppServerDriverOptions { /** Current server constraints; does not commit task status before the turn ends. */ completionFeedback?: (result: import("../../protocol/replay-contract.js").PrpStructuredRunResult) => Promise; environment?: NodeJS.ProcessEnv; + observability?: import("../agent-observability.js").ProviderObservabilityLaunch; /** Filesystem that authoritatively admits the workspace path. */ workingDirectoryAuthority?: CodexWorkingDirectoryAuthority; now?: () => Date; diff --git a/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts b/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts index 6832996fef5..76f312f4397 100644 --- a/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts +++ b/packages/paperclip-runner/src/drivers/runtime-context-materializer.ts @@ -14,6 +14,7 @@ import { } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { parse as parseToml } from "smol-toml"; +import { codexObservabilityToml } from "./agent-observability.js"; import type { NativeRuntimeContextSnapshot } from "../contracts/runtime-context.js"; import type { NativeMcpLaunchBinding } from "./native-mcp.js"; @@ -332,6 +333,7 @@ async function managedCodexProviderConfig(sourceHome?: string | null): Promise; options: CapabilityRunnerdCodexTransportOptions; identity: DurableRecoveryIdentity; @@ -3363,6 +3365,10 @@ export function createCapabilityRunnerdProviderEnvironment(input: { return environment; } +export function createCapabilityRunnerdProviderEnvironment(input: Parameters[0]): NodeJS.ProcessEnv { + return applyProviderObservabilityEnvironment(createCapabilityRunnerdProviderEnvironmentBase(input), input.options.observability, input.provider === "acpx"); +} + export function resolveRunnerdAcpxPermissionMode( configured: CapabilityRunnerdCodexTransportOptions["acpxPermissionMode"], ): NonNullable { @@ -4818,6 +4824,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await prepareIsolatedCodexHome({ context: sourceRuntimeContext, codexHome: localCodexHome, + observability: this.options.observability, sourceCodexHome: this.options.sourceCodexHome ?? resolveSourceCodexHome(this.options.environment), @@ -5763,6 +5770,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await prepareIsolatedCodexHome({ context: sourceRuntimeContext, codexHome: localCodexHome, + observability: this.options.observability, sourceCodexHome: this.options.sourceCodexHome ?? resolveSourceCodexHome(this.options.environment), @@ -7188,6 +7196,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } #diagnostic(message: string): void { + message = redactObservabilityDiagnostic(message, this.options.environment, this.options.observability?.otelConfig); this.#evidence.diagnostics.push(message); if (this.#evidence.diagnostics.length > 64) this.#evidence.diagnostics.shift(); diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json b/packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json new file mode 100644 index 00000000000..2991b8766d3 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json @@ -0,0 +1,51 @@ +{ + "commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "pi", + "agentProfileVersion": 28, + "acpxVersion": "0.13.1", + "agentServerVersion": "0.0.33", + "agentRuntimeVersion": "1.0.0", + "nodeVersion": "24.21.0", + "closure": { + "darwin-arm64": "282022db10150c6632b3444df421342e7d534bdf5d5fb1097a2e79d0625a2bcf", + "darwin-x64": "64e251e19009f755c0b04f73ce2138246faab71a961b0f13d75ebfcc34bef12e", + "linux-x64": "713b1fdff42fb56a1518bdc084f181d70bee8ebadc3e4b1d76321ed9108c8410" + }, + "wrapperSha256": "9d129b3d38772e93e97080aa6c4e574ac5df4e47ce5bc331a484a9fbf8188b36", + "helperSha256": "41e0490b617da0d60e0c8ec58ef311236f945129d99f816cff6897f78da7f91d", + "extensionSha256": "f5818573a355702388072b28926897db546a5cf346b5171f646582ce5f9102a4", + "agentFilesBinding": "registered-runtime-context-v1", + "nativeQuestions": "select-confirm-input-editor-v4-root-field-types", + "pathPolicy": "sdk-normalization-v1", + "mcpToolNames": "provider-safe-exact-source-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "nativeAssistantMessages": "pi1-rpc-deltas-v3-bound-message-and-tool-receipts", + "nativeNotices": "session-extension-v2-bounded-runtime-failure", + "messageProjectionSha256": "17d1e91b948dfa6a56e1905645add52b502041ad843c8d50a13b07956f22aadd", + "noticeProjectionSha256": "014485adc690998e395d334ea3ae67f72be6060a2fc78a1185df7b8d1da28fbf", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "nativeInputDisposition": "queued-only-v1", + "cacheWarming": "disabled-owned-decision-v1", + "thinkingLevel": { + "supported": [ + "off", + "low", + "high", + "max" + ], + "newConfigurationDefault": "low", + "admission": "explicit-native-effective-before-prompt-v1", + "recovery": "mode-bound-v1" + }, + "modelSelection": "caller-selected-native-verified-v1", + "providerConfigurationSourceSha256": "75a9a13f13f224dc092bd8ebf49e8747162172943761e44476c8eb18d80c52d4", + "credentialEnvironmentSourceSha256": "54fd09ad430dac04df98d829061f6701e73b5bf2a0f4de836d2ac36ee511009f", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "recoveryIdentitySourceSha256": "d2c9fc161fcc135fb9760110bfa3d984ee2bb38d87fa5ce7cb7418a49dd89e66", + "nativeCancellation": "acknowledged-cancelled-terminal-preserves-usage-v1", + "dependencySecurityPatchSha256": "5273a195b952f233336122842faaf650193c04f36b9359d20d4d63079b0ce814", + "openRouterAttributionSourceSha256": "1211191529d6073b733b311c3d69cada4973eede0f3e540c46e261f78e0b90f9" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v24-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v24-identity.json new file mode 100644 index 00000000000..8fcbfae75f7 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v24-identity.json @@ -0,0 +1,44 @@ +{ + "commandDigest": "sha256:87e275270e50b42290d10ecec9dfce86ffca2b7a7519cb60c584004adfc86437", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 24, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "fe1bb611bea289d81b1c7c973fbb01b93de133a7caa8fe78b7ff4ee1f485b224", + "permissionAdapterSourceSha256": "49421a779c76e286ce65d75052026eab9630b1995af9ccf045a2d43d284633ce", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "permissionIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "89e3dde25ff35bfd4d7f520fd9d14af3cb9027f182a2d51431d298f1015be580", + "toolEvidenceSourceSha256": "49d7d09eb6957e596a173b0afd763544afe582f0c1f05cc0992fe51916a8fdb3", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "6fc6ceb1222cec5586952cf51e2df045ecf751adfc937f5125550a311f5d3896", + "semanticDirectDriverSourceSha256": "72d5134a9567ced685bc942adf14ec5dfba04bcd6b0eeabe77eac0cec5c434c3", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "a104278d4d0e063a69bb464938db117dc27691b5bd4590c5a359e11f4917f623", + "semanticValidatorsSourceSha256": "38ae5379d47df633564730ec1359a16d21c0d0a68addb5e488b58c697f13362c", + "semanticSchemaBundleSourceSha256": "5e957fe0f0ba164441d6a5b8a276e52cf6f2ffaf99741faca99cff05e539221c", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "semanticContinuationSourceSha256": "eb877719ef32a935afa4ec93f0f1c43d58d55bc4e1042965febcac4c45ed851c" + } +} diff --git a/packages/shared/src/agent-observability.test.ts b/packages/shared/src/agent-observability.test.ts new file mode 100644 index 00000000000..36cd74a8a8d --- /dev/null +++ b/packages/shared/src/agent-observability.test.ts @@ -0,0 +1,21 @@ +import { expect, it } from "vitest"; +import { updateAgentObservabilitySchema, DEFAULT_AGENT_OBSERVABILITY_CAPTURE, normalizeOtlpBaseUrl } from "./agent-observability.js"; +const request = { expectedRevision: null, enabled: true, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example/otel/" }, credentials: { mode: "replace", headers: [] } }; +it("defaults capture and normalizes a path-prefixed anonymous collector", () => { + expect(updateAgentObservabilitySchema.parse(request)).toMatchObject({ capture: DEFAULT_AGENT_OBSERVABILITY_CAPTURE, destination: { endpointBaseUrl: "https://collector.example/otel" } }); +}); +it.each(["ftp://collector.example", "https://secret@collector.example", "https://collector.example?token=secret", "https://collector.example/#fragment", "https://collector.example/v1/logs"])("rejects ambiguous or credential-bearing endpoint %s", url => { + expect(() => normalizeOtlpBaseUrl(url)).toThrow(); +}); +it.each([ + [{ name: "Authorization", value: "valid" }, { name: "authorization", value: "duplicate" }], + [{ name: "host", value: "forged" }], [{ name: "x-auth", value: "a\r\nb" }], + [{ name: "x-auth", value: "[REDACTED]" }], [{ name: "x-auth", value: "é".repeat(2049) }], + [{ name: "x-auth", value: "\ud800" }], [{ name: "x-auth", value: "\udc00" }], +].map(headers => ({ headers })))("rejects invalid headers", ({ headers }) => { + expect(updateAgentObservabilitySchema.safeParse({ ...request, credentials: { mode: "replace", headers } }).success).toBe(false); +}); +it("rejects provider credential mismatches and unknown fields", () => { + expect(updateAgentObservabilitySchema.safeParse({ ...request, credentials: { mode: "replace", apiKey: "key" } }).success).toBe(false); + expect(updateAgentObservabilitySchema.safeParse({ ...request, injected: "value" }).success).toBe(false); +}); diff --git a/packages/shared/src/agent-observability.ts b/packages/shared/src/agent-observability.ts new file mode 100644 index 00000000000..4e71f2ca92f --- /dev/null +++ b/packages/shared/src/agent-observability.ts @@ -0,0 +1,96 @@ +import { z } from "zod"; + +export const DEFAULT_AGENT_OBSERVABILITY_CAPTURE = { + userPrompts: true, + assistantResponses: true, + toolDetails: true, + toolContent: true, +} as const; + +export const agentObservabilityCaptureSchema = z.strictObject({ + userPrompts: z.boolean(), assistantResponses: z.boolean(), + toolDetails: z.boolean(), toolContent: z.boolean(), +}); +export type AgentObservabilityCapture = z.infer; + +export function normalizeOtlpBaseUrl(value: string): string { + const url = new URL(value); + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) { + throw new Error("Use an HTTP or HTTPS base URL without credentials, query, or fragment."); + } + url.pathname = url.pathname.replace(/\/+$/, ""); + if (/\/v1\/(?:logs|traces|metrics)$/i.test(url.pathname)) { + throw new Error("Enter the OTLP base URL, without /v1/logs or /v1/traces."); + } + return url.toString().replace(/\/+$/, ""); +} + +const baseUrl = z.string().max(2048).transform((value, ctx) => { + try { return normalizeOtlpBaseUrl(value); } + catch { ctx.addIssue({ code: "custom", message: "Enter an HTTP or HTTPS OTLP base URL without credentials, query, fragment, or signal path." }); return z.NEVER; } +}); +export const agentObservabilityDestinationSchema = z.discriminatedUnion("provider", [ + z.strictObject({ provider: z.literal("braintrust"), projectId: z.string().uuid() }), + z.strictObject({ provider: z.literal("otlp"), endpointBaseUrl: baseUrl }), +]); +export type AgentObservabilityDestination = z.infer; + +/** Saved custom credentials belong to their HTTP origin, including scheme and port. */ +export function isSameOtlpOrigin(previous: AgentObservabilityDestination, next: AgentObservabilityDestination): boolean { + if (previous.provider !== "otlp" || next.provider !== "otlp") return false; + try { return new URL(previous.endpointBaseUrl).origin === new URL(next.endpointBaseUrl).origin; } + catch { return false; } +} + +const forbiddenHeaders = new Set(["host", "content-length", "content-type", "connection", "transfer-encoding", "accept", "accept-encoding"]); +const headerName = z.string().min(1).max(128).regex(/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/) + .transform(value => value.toLowerCase()).refine(value => !forbiddenHeaders.has(value), "This header is managed by the transport."); +const headerValue = z.string().min(1).refine(value => !/[\x00-\x1f\x7f-\x9f]/.test(value), "Header values cannot contain control characters.") + .refine(value => { try { encodeURIComponent(value); return true; } catch { return false; } }, "Header values must contain valid Unicode.") + .refine(value => new TextEncoder().encode(value).length <= 4096, "Header values must be at most 4096 bytes.") + .refine(value => !/^(?:\*{3,}|•{3,}||\[redacted\])$/i.test(value), "Enter the credential rather than a masked placeholder."); +export const agentObservabilityHeadersSchema = z.array(z.strictObject({ name: headerName, value: headerValue.optional() })).max(16) + .superRefine((headers, ctx) => { + if (new Set(headers.map(row => row.name)).size !== headers.length) ctx.addIssue({ code: "custom", message: "Header names must be unique." }); + if (headers.reduce((sum, row) => sum + new TextEncoder().encode(row.name + (row.value ?? "")).length, 0) > 16384) { + ctx.addIssue({ code: "custom", message: "Headers must total at most 16 KiB." }); + } + }); +const common = { + expectedRevision: z.string().uuid().nullable(), enabled: z.boolean(), + capture: agentObservabilityCaptureSchema.default(DEFAULT_AGENT_OBSERVABILITY_CAPTURE), +}; +const keep = z.strictObject({ mode: z.literal("keep") }); +export const updateAgentObservabilitySchema = z.union([ + z.strictObject({ ...common, + destination: z.strictObject({ provider: z.literal("braintrust"), projectId: z.string().uuid() }), + credentials: z.discriminatedUnion("mode", [keep, z.strictObject({ mode: z.literal("replace"), apiKey: headerValue })]), + }), + z.strictObject({ ...common, + destination: z.strictObject({ provider: z.literal("otlp"), endpointBaseUrl: baseUrl }), + credentials: z.discriminatedUnion("mode", [keep, z.strictObject({ mode: z.literal("replace"), headers: agentObservabilityHeadersSchema })]), + }), +]); +export type UpdateAgentObservabilityRequest = z.infer; +export const agentObservabilityRevisionSchema = z.strictObject({ expectedRevision: z.string().uuid() }); +export type OtlpSignalTestResult = { + status: "accepted" | "rejected" | "failed"; httpStatus: number | null; + rejectedRecords: number; errorCode: string | null; message: string | null; durationMs: number; +}; +export type AgentObservabilityTestReceipt = { + revision: string; marker: string; source: "control_plane"; startedAt: string; finishedAt: string; + status: "accepted" | "partial" | "failed"; + signals: { logs: OtlpSignalTestResult; traces: OtlpSignalTestResult }; +}; +export type AgentObservabilitySettings = { + companyId: string; revision: string; enabled: boolean; destination: AgentObservabilityDestination; + capture: AgentObservabilityCapture; credentialsConfigured: boolean; headerNames: string[]; + updatedAt: string; lastTest: AgentObservabilityTestReceipt | null; +}; +export type AgentObservabilitySettingsResponse = { canManage: boolean; settings: AgentObservabilitySettings | null }; + +export type AgentObservabilityUnavailableReason = + | "configuration_invalid" | "credentials_unavailable" | "unsupported_harness" | "unsupported_home_mode"; +export type AgentObservabilityErrorCode = + | "observability_revision_conflict" | "observability_credentials_required" | "observability_header_value_required" + | AgentObservabilityUnavailableReason | "timeout" | "network_error" | "receiver_rejected" | "partial_rejection" | "invalid_response"; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 1b82d5534a5..1b1f1e82cb4 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2828,3 +2828,4 @@ export * from "./fast-response.js"; export { HUMAN_TASK_ORIGIN_KINDS, isAgentCreatedIssue, type IssueCreationSourceFields } from "./issue-creation-source.js"; export * from "./grok-bot.js"; +export * from "./agent-observability.js"; diff --git a/scripts/pi-public-install-probe.mjs b/scripts/pi-public-install-probe.mjs index 4fa24c5c6bd..5e6672cc68f 100644 --- a/scripts/pi-public-install-probe.mjs +++ b/scripts/pi-public-install-probe.mjs @@ -52,8 +52,8 @@ assert.equal(JSON.parse(await readFile(join(server, 'package.json'), 'utf8')).na const { resolvePaperclipRunnerBinary } = await import(pathToFileURL(join(server, 'dist/services/native-runtime/native-codex-runner.js'))); const { createCapabilityRunnerdCodexTransport } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/live/runnerd-codex-transport.js'))); const { QUALIFIED_ACPX_PROFILES } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/drivers/acpx/qualified-profiles.js'))); -assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 27); -assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:b1b75a1ae3ac498eb07b95b96bebf4da86227b83d09276076dd5c222e3964dba'); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 28); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c'); assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'qualificationModel'), false); assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'reportedModelId'), false); const daemon = resolvePaperclipRunnerBinary(); diff --git a/server/package.json b/server/package.json index 8ef255259c5..587c679dc1c 100644 --- a/server/package.json +++ b/server/package.json @@ -63,6 +63,7 @@ "@modelcontextprotocol/sdk": "^1.31.0", "@openrouter/ai-sdk-provider": "3.1.0", "@opentelemetry/api": "^1.9.0", + "@opentelemetry/otlp-transformer": "0.221.0", "@paperclipai/adapter-claude-local": "workspace:*", "@paperclipai/adapter-codex-local": "workspace:*", "@paperclipai/adapter-cursor-cloud": "workspace:*", @@ -112,7 +113,8 @@ "ssh2": "^1.17.0", "svix": "1.76.1", "ws": "^8.22.0", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@opentelemetry/resources": "2.10.0" }, "bundleDependencies": [ "@agentclientprotocol/claude-agent-acp", @@ -143,7 +145,8 @@ "tsx": "^4.23.15", "typescript": "^7.0.2", "vite": "^8.2.2", - "vitest": "^5.0.3" + "vitest": "^5.0.3", + "protobufjs": "7.6.6" }, "peerDependencies": { "@opentelemetry/auto-instrumentations-node": "0.79.0", diff --git a/server/src/__tests__/agent-observability-test.test.ts b/server/src/__tests__/agent-observability-test.test.ts new file mode 100644 index 00000000000..d56f59b0062 --- /dev/null +++ b/server/src/__tests__/agent-observability-test.test.ts @@ -0,0 +1,88 @@ +import { afterAll, beforeAll, expect, it } from "vitest"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { readFile } from "node:fs/promises"; +import protobuf from "protobufjs"; +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE } from "@paperclipai/shared"; +import { testAgentObservability } from "../services/agent-observability-test.js"; + +let server: Server, endpoint: string; +let responseMode = "success"; +const requests: Array<{ path: string; headers: Record; body: Buffer }> = []; +let root: protobuf.Root; +beforeAll(async () => { + root = protobuf.parse(await readFile(new URL("./fixtures/otlp-observability.proto", import.meta.url), "utf8")).root; + server = createServer(async (req, res) => { + // Fault scenarios must not leave pooled sockets for the next scenario. + res.setHeader("connection", "close"); + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + requests.push({ path: req.url!, headers: req.headers, body: Buffer.concat(chunks) }); + if (responseMode === "hang") return; + if (responseMode === "body-hang") { res.writeHead(200, { "content-type": "application/x-protobuf" }); res.write(Buffer.from([10])); return; } + if (responseMode === "delayed") { setTimeout(() => { res.writeHead(200); res.end(); }, 150); return; } + if (responseMode === "oversize") { res.writeHead(200, { "content-type": "application/x-protobuf" }); res.end(Buffer.alloc(65537)); return; } + if (/^status-\d+$/.test(responseMode)) { res.writeHead(Number(responseMode.slice(7))); res.end("untrusted-secret-error"); return; } + if (responseMode === "reject") { res.writeHead(403); res.end("secret provider diagnostic must not escape"); return; } + if (responseMode === "malformed") { res.writeHead(200, { "content-type": "application/json" }); res.end("malformed"); return; } + if (responseMode === "bad-shape") { res.writeHead(200, { "content-type": "application/json" }); res.end('{"partialSuccess":"invalid"}'); return; } + if (responseMode === "html") { res.writeHead(200, { "content-type": "text/html" }); res.end(); return; } + if (responseMode === "json-partial") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ partialSuccess: { [req.url?.endsWith("logs") ? "rejectedLogRecords" : "rejectedSpans"]: "1", errorMessage: "untrusted-secret-error" } })); return; } + if (responseMode === "partial") { + res.writeHead(200, { "content-type": "application/x-protobuf" }); + const type = root.lookupType("paperclip.test.otlp.ExportResponse"); + res.end(type.encode(type.create({ partialSuccess: { rejectedLogRecords: 1, errorMessage: "untrusted-secret-error" } })).finish()); return; + } + res.writeHead(200, { "content-type": "application/x-protobuf" }); res.end(); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); +afterAll(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); }); +function run(timeoutMs?: number, allowPrivateNetwork = true) { + return testAgentObservability({ companyId: "10000000-0000-4000-8000-000000000001", revision: "20000000-0000-4000-8000-000000000001", timeoutMs, allowPrivateNetwork, + resolved: { state: "enabled", revision: "20000000-0000-4000-8000-000000000001", compatibilityKey: "test", logsEndpoint: `${endpoint}/otel/v1/logs`, tracesEndpoint: `${endpoint}/otel/v1/traces`, + headers: { authorization: "Bearer local-fixture", "x-bt-parent": "project_id:fixture" }, capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE } } }); +} +it("sends actual protobuf records with one root span, correlated IDs, and exact routing", async () => { + responseMode = "success"; requests.length = 0; + const result = await run(); expect(result.status).toBe("accepted"); + expect(requests.map(row => row.path).sort()).toEqual(["/otel/v1/logs", "/otel/v1/traces"]); + for (const req of requests) expect(req.headers).toMatchObject({ authorization: "Bearer local-fixture", "x-bt-parent": "project_id:fixture", "content-type": "application/x-protobuf" }); + const logs = root.lookupType("paperclip.test.otlp.ExportLogsServiceRequest").decode(requests.find(row => row.path.endsWith("logs"))!.body).toJSON(); + const traces = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(requests.find(row => row.path.endsWith("traces"))!.body).toJSON(); + const log = logs.resourceLogs[0].scopeLogs[0].logRecords[0], span = traces.resourceSpans[0].scopeSpans[0].spans[0]; + expect(log.body.stringValue).toBe(result.marker); expect(log.traceId).toBe(span.traceId); expect(log.spanId).toBe(span.spanId); + expect(span.parentSpanId).toBeUndefined(); expect(span.name).toBe("paperclip.observability.connection_test"); +}); +it.each(["reject", "malformed", "bad-shape", "html", "partial", "json-partial"])("reports %s without echoing receiver data", async mode => { + responseMode = mode; const result = await run(); + expect(result.status).toBe(mode.includes("partial") ? "partial" : "failed"); + expect(JSON.stringify(result)).not.toContain("untrusted-secret"); expect(JSON.stringify(result)).not.toContain("must not escape"); +}); +it("aborts a receiver that accepts connections and never responds", async () => { + responseMode = "hang"; const start = performance.now(); const result = await run(100); + expect(result.status).toBe("failed"); expect(result.signals.logs.errorCode).toBe("timeout"); + expect(performance.now() - start).toBeLessThan(2000); server.closeAllConnections(); +}); +it.each([401, 403, 404, 429, 500])("sanitizes HTTP %s without retrying", async status => { + responseMode = `status-${status}`; requests.length = 0; + const result = await run(); + expect(requests).toHaveLength(2); + expect(result.signals.logs).toMatchObject({ status: "failed", httpStatus: status, errorCode: "receiver_rejected" }); + expect(JSON.stringify(result)).not.toContain("untrusted-secret"); +}); +it.each(["body-hang", "delayed"])("uses the shared deadline for %s", async mode => { + responseMode = mode; + const result = await run(50); + expect(result.signals.logs.errorCode).toBe("timeout"); + expect(result.signals.traces.errorCode).toBe("timeout"); + server.closeAllConnections(); +}); +it("rejects a response larger than 64 KiB", async () => { + responseMode = "oversize"; + expect((await run()).status).toBe("failed"); +}); +it("denies private endpoints in public mode without delivering a request", async () => { + responseMode = "success"; requests.length = 0; + expect((await run(1000, false)).status).toBe("failed"); expect(requests).toHaveLength(0); +}); diff --git a/server/src/__tests__/agent-observability.test.ts b/server/src/__tests__/agent-observability.test.ts new file mode 100644 index 00000000000..06cc18df617 --- /dev/null +++ b/server/src/__tests__/agent-observability.test.ts @@ -0,0 +1,243 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq, sql } from "drizzle-orm"; +import * as codexServer from "@paperclipai/adapter-codex-local/server"; +import express from "express"; +import request from "supertest"; +import { createDb, companies, companyMemberships, companyObservability, companySecrets, companySecretBindings, activityLog } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE, type UpdateAgentObservabilityRequest } from "@paperclipai/shared"; +import { agentObservabilityService } from "../services/agent-observability.js"; +import { deleteCompany } from "../services/company-deletion.js"; +import { agentObservabilityRoutes } from "../routes/agent-observability.js"; +import { secretService } from "../services/secrets.js"; +import * as secretsModule from "../services/secrets.js"; +import { errorHandler } from "../middleware/index.js"; + +let database: Awaited>; +let db: ReturnType, home: string; +beforeAll(async () => { + home = await realpath(await mkdtemp(join(tmpdir(), "paperclip-observability-"))); + vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "observability-test"); + database = await startEmbeddedPostgresTestDatabase("paperclip-observability-db-"); + db = createDb(database.connectionString); +}, 90000); +afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); +let sequence = 0; +async function fixture() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Observability fixture", issuePrefix: `OB${++sequence}` }); + await db.insert(companyMemberships).values({ companyId, principalId: "alice", principalType: "user", status: "active", membershipRole: "owner" }); + return { companyId, service: agentObservabilityService(db) }; +} +const braintrust = (expectedRevision: string | null = null): UpdateAgentObservabilityRequest => ({ expectedRevision, + enabled: true, capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, destination: { provider: "braintrust", projectId: randomUUID() }, + credentials: { mode: "replace", apiKey: "observability-secret-canary-123456" } }); + +describe("company agent observability", () => { + it("creates atomically, returns no credentials, and rejects stale writes before rotation", async () => { + const f = await fixture(); + const outcomes = await Promise.allSettled([f.service.configure(f.companyId, "alice", braintrust()), f.service.configure(f.companyId, "alice", braintrust())]); + expect(outcomes.filter(row => row.status === "fulfilled")).toHaveLength(1); + expect(outcomes.find(row => row.status === "rejected")).toMatchObject({ reason: { status: 409 } }); + const saved = (await f.service.settings(f.companyId))!; + expect(JSON.stringify(saved)).not.toContain("canary"); + expect(saved).not.toHaveProperty("credentialsSecretId"); + const [secret] = await db.select().from(companySecrets).where(eq(companySecrets.companyId, f.companyId)); + expect(secret.latestVersion).toBe(1); + await expect(f.service.configure(f.companyId, "alice", braintrust())).rejects.toMatchObject({ status: 409 }); + expect((await secretService(db).getById(secret.id))?.latestVersion).toBe(1); + const resolved = await f.service.resolve(f.companyId); + expect(resolved).toMatchObject({ state: "enabled", logsEndpoint: "https://api.braintrust.dev/otel/v1/logs", headers: { authorization: "Bearer observability-secret-canary-123456", "x-bt-parent": `project_id:${saved.destination.provider === "braintrust" ? saved.destination.projectId : ""}` } }); + const audit = await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId)); + expect(JSON.stringify(audit)).not.toContain("canary"); + }); + it("retains, updates, and removes write-only custom headers", async () => { + const f = await fixture(); + let saved = await f.service.configure(f.companyId, "alice", { expectedRevision: null, enabled: true, capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, + destination: { provider: "otlp", endpointBaseUrl: "https://collector.example.com/otel/" }, + credentials: { mode: "replace", headers: [{ name: "Authorization", value: "Bearer keep-me" }, { name: "X-Old", value: "remove-me" }] } }); + saved = await f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, destination: saved.destination as { provider: "otlp"; endpointBaseUrl: string }, + credentials: { mode: "replace", headers: [{ name: "AUTHORIZATION" }, { name: "x-new", value: "new-value" }] } }); + const resolved = await f.service.resolve(f.companyId); + expect(resolved).toMatchObject({ state: "enabled", logsEndpoint: "https://collector.example.com/otel/v1/logs", headers: { authorization: "Bearer keep-me", "x-new": "new-value" } }); + if (resolved.state === "enabled") expect(resolved.headers).not.toHaveProperty("x-old"); + await expect(f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, destination: saved.destination as { provider: "otlp"; endpointBaseUrl: string }, + credentials: { mode: "replace", headers: [{ name: "renamed" }] } })).rejects.toMatchObject({ status: 422 }); + await f.service.remove(f.companyId, "alice", saved.revision); + expect(await f.service.settings(f.companyId)).toBeNull(); + expect(await db.select().from(companySecretBindings).where(eq(companySecretBindings.companyId, f.companyId))).toEqual([]); + expect(await db.select().from(companySecrets).where(eq(companySecrets.companyId, f.companyId))).toEqual([]); + }); + it.each(["save", "remove", "delete-company"])("acquires secret mutation ownership before database locks: %s", async operation => { + const f = await fixture(); + const saved = await f.service.configure(f.companyId, "alice", { expectedRevision: null, enabled: true, + capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example" }, + credentials: { mode: "replace", headers: [{ name: "authorization", value: "retained-value" }] } }); + const [row] = await db.select().from(companyObservability).where(eq(companyObservability.companyId, f.companyId)); + const original = codexServer.withAccountHomeSecretMutationLock; + let probed = false; + const spy = vi.spyOn(codexServer, "withAccountHomeSecretMutationLock").mockImplementation(async (env, companyId, work) => { + if (companyId === f.companyId && !probed) { + probed = true; + // A separate real database connection must acquire these rows NOWAIT + // at the first mutation-lock attempt. The old order already holds them. + await db.transaction(async tx => { + await tx.execute(sql`select 1 from companies where id = ${companyId} for no key update nowait`); + await tx.execute(sql`select 1 from company_secrets where id = ${row.credentialsSecretId} for update nowait`); + }); + } + return original(env, companyId, work); + }); + try { + if (operation === "save") { + await f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, + destination: saved.destination as { provider: "otlp"; endpointBaseUrl: string }, + credentials: { mode: "replace", headers: [{ name: "authorization" }, { name: "x-new", value: "new-value" }] } }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ headers: { authorization: "retained-value", "x-new": "new-value" } }); + } else { + if (operation === "remove") await f.service.remove(f.companyId, "alice", saved.revision); + else await deleteCompany(db, f.companyId); + expect(await f.service.settings(f.companyId)).toBeNull(); + expect(await secretService(db).getById(row.credentialsSecretId!)).toBeNull(); + } + expect(probed).toBe(true); + } finally { spy.mockRestore(); } + }); + + it("completes concurrent header retention and external credential rotation", async () => { + const f = await fixture(); + const saved = await f.service.configure(f.companyId, "alice", { expectedRevision: null, enabled: true, + capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example" }, + credentials: { mode: "replace", headers: [{ name: "authorization", value: "original-value" }] } }); + const [row] = await db.select().from(companyObservability).where(eq(companyObservability.companyId, f.companyId)); + await Promise.all([ + f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, + destination: saved.destination as { provider: "otlp"; endpointBaseUrl: string }, credentials: { mode: "replace", headers: [{ name: "authorization" }] } }), + secretService(db).rotate(row.credentialsSecretId!, { value: JSON.stringify({ schema: "paperclip.agent_observability.credentials.v1", headers: { authorization: "rotated-value" } }) }), + ]); + expect((await secretService(db).getById(row.credentialsSecretId!))?.latestVersion).toBe(3); + expect(await f.service.resolve(f.companyId)).toMatchObject({ headers: { authorization: "rotated-value" } }); + }); + + it("settles concurrent save and removal through revision conflict without a lock cycle", async () => { + const f = await fixture(); const saved = await f.service.configure(f.companyId, "alice", braintrust()); + const results = await Promise.allSettled([ + f.service.configure(f.companyId, "alice", { ...braintrust(saved.revision), enabled: false, credentials: { mode: "keep" } }), + f.service.remove(f.companyId, "alice", saved.revision), + ]); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + expect(results.find(result => result.status === "rejected")).toMatchObject({ reason: { status: 409 } }); + }); + + it("settles a concurrent settings save and real company deletion without a lock cycle", async () => { + const f = await fixture(); const saved = await f.service.configure(f.companyId, "alice", braintrust()); + const results = await Promise.allSettled([ + f.service.configure(f.companyId, "alice", { ...braintrust(saved.revision), enabled: false, credentials: { mode: "keep" } }), + deleteCompany(db, f.companyId), + ]); + expect(results[1].status).toBe("fulfilled"); + if (results[0].status === "rejected") expect(results[0].reason).toMatchObject({ status: 404 }); + expect(await f.service.settings(f.companyId)).toBeNull(); + expect(await db.select().from(companySecrets).where(eq(companySecrets.companyId, f.companyId))).toEqual([]); + }); + + it("keeps project routing current and changes process compatibility on external key rotation", async () => { + const f = await fixture(); const saved = await f.service.configure(f.companyId, "alice", braintrust()); + const before = await f.service.resolve(f.companyId); + const [row] = await db.select().from(companyObservability).where(eq(companyObservability.companyId, f.companyId)); + await secretService(db).rotate(row.credentialsSecretId!, { value: JSON.stringify({ schema: "paperclip.agent_observability.credentials.v1", headers: { authorization: "Bearer rotated" } }) }); + const rotated = await f.service.resolve(f.companyId); + expect(rotated.compatibilityKey).not.toBe(before.compatibilityKey); + const projectId = randomUUID(); + await f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, destination: { provider: "braintrust", projectId }, credentials: { mode: "keep" } }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ headers: { authorization: "Bearer rotated", "x-bt-parent": `project_id:${projectId}` } }); + }); + it("binds retained custom credentials to their original endpoint origin", async () => { + const f = await fixture(); + const saved = await f.service.configure(f.companyId, "alice", { expectedRevision: null, enabled: true, capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, + destination: { provider: "otlp", endpointBaseUrl: "https://collector.example/otel" }, + credentials: { mode: "replace", headers: [{ name: "authorization", value: "old-origin-credential" }] } }); + const [row] = await db.select().from(companyObservability).where(eq(companyObservability.companyId, f.companyId)); + const auditsBefore = await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId)); + for (const endpointBaseUrl of ["https://another.example", "http://collector.example", "https://collector.example:8443"]) { + for (const credentials of [{ mode: "keep" as const }, { mode: "replace" as const, headers: [{ name: "authorization" }] }]) { + await expect(f.service.configure(f.companyId, "another-manager", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, + destination: { provider: "otlp", endpointBaseUrl }, credentials })).rejects.toMatchObject({ status: 422 }); + } + } + expect((await f.service.settings(f.companyId))!.revision).toBe(saved.revision); + expect((await secretService(db).getById(row.credentialsSecretId!))?.latestVersion).toBe(1); + expect(await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId))).toEqual(auditsBefore); + const sameOrigin = await f.service.configure(f.companyId, "alice", { expectedRevision: saved.revision, enabled: true, capture: saved.capture, + destination: { provider: "otlp", endpointBaseUrl: "https://collector.example/new-prefix" }, credentials: { mode: "keep" } }); + const replaced = await f.service.configure(f.companyId, "alice", { expectedRevision: sameOrigin.revision, enabled: true, capture: saved.capture, + destination: { provider: "otlp", endpointBaseUrl: "https://another.example" }, credentials: { mode: "replace", headers: [{ name: "authorization", value: "new-origin-credential" }] } }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ state: "enabled", logsEndpoint: "https://another.example/v1/logs", headers: { authorization: "new-origin-credential" } }); + await f.service.configure(f.companyId, "alice", { expectedRevision: replaced.revision, enabled: true, capture: saved.capture, + destination: { provider: "otlp", endpointBaseUrl: "https://anonymous.example" }, credentials: { mode: "replace", headers: [] } }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ state: "enabled", headers: {} }); + }); + it("uses the selected numeric credential version even when latest rotates before value resolution", async () => { + const f = await fixture(); await f.service.configure(f.companyId, "alice", braintrust()); + const before = await f.service.resolve(f.companyId); + const original = secretsModule.secretService; + const spy = vi.spyOn(secretsModule, "secretService").mockImplementation(executor => { + const service = original(executor); + return { ...service, resolveSecretVersion: async (...args: Parameters) => { + const version = await service.resolveSecretVersion(...args); + await original(db).rotate(args[1], { value: JSON.stringify({ schema: "paperclip.agent_observability.credentials.v1", headers: { authorization: "Bearer concurrently-rotated" } }) }); + return version; + } }; + }); + try { + const resolved = await f.service.resolve(f.companyId); + expect(resolved).toMatchObject({ state: "enabled", headers: { authorization: "Bearer observability-secret-canary-123456" } }); + expect(resolved.compatibilityKey).toBe(before.compatibilityKey); + } finally { spy.mockRestore(); } + expect(await f.service.resolve(f.companyId)).toMatchObject({ headers: { authorization: "Bearer concurrently-rotated" } }); + }); + it("can disable and remove when credentials are unavailable", async () => { + const f = await fixture(); const saved = await f.service.configure(f.companyId, "alice", braintrust()); + const [row] = await db.select().from(companyObservability).where(eq(companyObservability.companyId, f.companyId)); + await secretService(db).update(row.credentialsSecretId!, { status: "disabled" }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ state: "unavailable", reason: "credentials_unavailable" }); + const disabled = await f.service.configure(f.companyId, "alice", { ...braintrust(saved.revision), enabled: false, destination: saved.destination as { provider: "braintrust"; projectId: string }, credentials: { mode: "keep" } }); + expect(await f.service.resolve(f.companyId)).toMatchObject({ state: "disabled" }); + await f.service.remove(f.companyId, "alice", disabled.revision); + }); + it("does not attach an old test receipt after another administrator saves", async () => { + const f = await fixture(); const saved = await f.service.configure(f.companyId, "alice", braintrust()); + const updated = await f.service.configure(f.companyId, "alice", { ...braintrust(saved.revision), enabled: false, credentials: { mode: "keep" } }); + const signal = { status: "accepted" as const, httpStatus: 200, rejectedRecords: 0, errorCode: null, message: null, durationMs: 1 }; + await expect(f.service.recordTest(f.companyId, "alice", { revision: saved.revision, marker: "stale-test", source: "control_plane", startedAt: new Date().toISOString(), finishedAt: new Date().toISOString(), status: "accepted", signals: { logs: signal, traces: signal } })).rejects.toMatchObject({ status: 409 }); + expect(await f.service.settings(f.companyId)).toMatchObject({ revision: updated.revision, lastTest: null }); + }); + it("retires feature credentials inside the company deletion transaction", async () => { + const f = await fixture(); await f.service.configure(f.companyId, "alice", braintrust()); + await deleteCompany(db, f.companyId); + expect(await f.service.settings(f.companyId)).toBeNull(); + expect(await db.select().from(companySecretBindings).where(eq(companySecretBindings.companyId, f.companyId))).toEqual([]); + expect(await db.select().from(companySecrets).where(eq(companySecrets.companyId, f.companyId))).toEqual([]); + }); + it("enforces board, company, and management permissions before accepting mutation payloads", async () => { + const f = await fixture(), other = await fixture(); + const app = express(); app.use(express.json()); + let actor: Record = { type: "board", userId: "alice", source: "session", companyIds: [f.companyId], memberships: [{ companyId: f.companyId, status: "active", membershipRole: "owner" }] }; + app.use((req, _res, next) => { req.actor = actor as typeof req.actor; next(); }); + app.use("/api", agentObservabilityRoutes(db)); app.use(errorHandler); + const url = `/api/companies/${f.companyId}/observability`; + const created = await request(app).put(url).send(braintrust()); expect(created.status).toBe(200); + expect(JSON.stringify(created.body)).not.toContain("canary"); + expect((await request(app).get(`/api/companies/${other.companyId}/observability`)).status).toBe(403); + actor = { ...actor, memberships: [{ companyId: f.companyId, status: "active", membershipRole: "viewer" }] }; + expect((await request(app).get(url)).body).toEqual({ canManage: false, settings: null }); + expect((await request(app).post(`${url}/test`).send({})).status).toBe(403); + actor = { type: "agent", agentId: randomUUID(), companyId: f.companyId, source: "agent_key" }; + expect((await request(app).get(url)).status).toBe(403); + expect((await request(app).delete(url).send({ expectedRevision: created.body.settings.revision })).status).toBe(403); + }); +}); diff --git a/server/src/__tests__/artifact-review-document-routes.test.ts b/server/src/__tests__/artifact-review-document-routes.test.ts index 21dd1ae60fd..2a34aa2efb0 100644 --- a/server/src/__tests__/artifact-review-document-routes.test.ts +++ b/server/src/__tests__/artifact-review-document-routes.test.ts @@ -137,10 +137,10 @@ function createStorageService(): StorageService { } async function createApp(options?: { companyIds?: string[] }) { - const [{ errorHandler }, { issueRoutes }] = await Promise.all([ - vi.importActual("../middleware/index.js"), - vi.importActual("../routes/issues.js"), - ]); + // Both modules traverse the reset/mocked service graph; concurrent imports + // can wait on each other's partially evaluated modules. + const { errorHandler } = await vi.importActual("../middleware/index.js"); + const { issueRoutes } = await vi.importActual("../routes/issues.js"); const app = express(); app.use(express.json()); app.use((req, _res, next) => { diff --git a/server/src/__tests__/error-handler.test.ts b/server/src/__tests__/error-handler.test.ts index c75952fc845..a2d36a49765 100644 --- a/server/src/__tests__/error-handler.test.ts +++ b/server/src/__tests__/error-handler.test.ts @@ -221,12 +221,16 @@ describe("errorHandler", () => { expect(telemetryMocks.trackErrorHandlerCrash).toHaveBeenCalledTimes(2); }); - it("sanitizes chat setup errors before logs and crash reporting", () => { + it.each([ + { name: "chat setup", method: "POST", path: "/api/chat-endpoints/endpoint-1/setup", body: { credentials: { botToken: "setup-error-token-canary" } } }, + { name: "observability API keys", method: "PUT", path: "/api/companies/company/observability", body: { credentials: { mode: "replace", apiKey: "setup-error-token-canary" } } }, + { name: "arbitrary OTLP headers", method: "PUT", path: "/api/companies/company/observability", body: { credentials: { mode: "replace", headers: [{ name: "x-custom", value: "setup-error-token-canary" }] } } }, + ])("sanitizes $name errors before logs and crash reporting", ({ method, path, body }) => { const req = { ...makeReq(), - method: "POST", - originalUrl: "/api/chat-endpoints/endpoint-1/setup", - body: { credentials: { botToken: "setup-error-token-canary" } }, + method, + originalUrl: path, + body, } as unknown as Request; const res = makeRes() as any; const next = vi.fn() as unknown as NextFunction; diff --git a/server/src/__tests__/fixtures/otlp-observability.proto b/server/src/__tests__/fixtures/otlp-observability.proto new file mode 100644 index 00000000000..ed258755a2d --- /dev/null +++ b/server/src/__tests__/fixtures/otlp-observability.proto @@ -0,0 +1,21 @@ +// Wire-compatible test subset of OpenTelemetry protobuf messages. +// Field numbers/types: https://github.com/open-telemetry/opentelemetry-proto/tree/v1.9.0/opentelemetry/proto +// Unknown production fields are intentionally ignored by this test decoder. +syntax = "proto3"; +package paperclip.test.otlp; +message AnyValue { oneof value { string string_value = 1; bool bool_value = 2; int64 int_value = 3; double double_value = 4; ArrayValue array_value = 5; KeyValueList kvlist_value = 6; bytes bytes_value = 7; } } +message ArrayValue { repeated AnyValue values = 1; } +message KeyValueList { repeated KeyValue values = 1; } +message KeyValue { string key = 1; AnyValue value = 2; } +message Resource { repeated KeyValue attributes = 1; } +message Scope { string name = 1; string version = 2; } +message LogRecord { fixed64 time_unix_nano = 1; int32 severity_number = 2; string severity_text = 3; AnyValue body = 5; repeated KeyValue attributes = 6; fixed32 flags = 8; bytes trace_id = 9; bytes span_id = 10; fixed64 observed_time_unix_nano = 11; string event_name = 12; } +message ScopeLogs { Scope scope = 1; repeated LogRecord log_records = 2; } +message ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2; } +message ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1; } +message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; } +message ScopeSpans { Scope scope = 1; repeated Span spans = 2; } +message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; } +message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; } +message PartialSuccess { int64 rejected_log_records = 1; string error_message = 2; } +message ExportResponse { PartialSuccess partial_success = 1; } diff --git a/server/src/__tests__/http-log-policy.test.ts b/server/src/__tests__/http-log-policy.test.ts index 850d1254e2f..5d717089938 100644 --- a/server/src/__tests__/http-log-policy.test.ts +++ b/server/src/__tests__/http-log-policy.test.ts @@ -64,6 +64,14 @@ describe("isPrivateWebhookHttpRequest", () => { }); describe("isSecretSensitiveHttpRequest", () => { + it("protects company observability credentials on accepted and rejected paths", () => { + for (const method of ["GET", "PUT", "POST", "DELETE", "PATCH"]) { + for (const url of ["/api/companies/company/observability", "/api/companies/company/observability/test", "/api/companies/company/observability/unknown", "https://paperclip.test/API/COMPANIES/company/OBSERVABILITY?ignored=private"]) { + expect(isSecretSensitiveHttpRequest(method, url)).toBe(true); + } + } + expect(isSecretSensitiveHttpRequest("PUT", "/api/companies/company/observability-extra")).toBe(false); + }); it("protects Slack registration, installation, recovery, and all callback methods", () => { for (const suffix of ["registration", "install", "resume"]) expect(isSecretSensitiveHttpRequest("POST", `/api/chat-endpoints/id/slack/${suffix}`)).toBe(true); for (const method of ["GET", "POST", "DELETE"]) for (const path of ["/api/chat-slack/oauth/callback?code=canary", "/api/chat-slack/managed/oauth/callback?claim=canary", "/api/chat-slack/managed/oauth/callback?code=canary", "/api/chat-endpoints/endpoint/slack/managed/authorize", "/api/chat-endpoints/endpoint/slack/managed/provision", "https://paperclip.test/API/CHAT-SLACK/OAUTH/callback?code=canary"]) diff --git a/server/src/__tests__/http-log-redaction.test.ts b/server/src/__tests__/http-log-redaction.test.ts index edefe3fac82..9e075c7c9f4 100644 --- a/server/src/__tests__/http-log-redaction.test.ts +++ b/server/src/__tests__/http-log-redaction.test.ts @@ -12,7 +12,7 @@ import { testAdapterEnvironmentSchema } from "@paperclipai/shared"; import { createHttpLogger } from "../middleware/logger.js"; describe("HTTP logger redaction", () => { - it.each([200, 400, 500])("keeps Slack setup credentials and provider echoes out of %i logs", async status => { + it.each([200, 400, 500])("keeps connector and observability credentials out of %i logs", async status => { const canaries = ["configuration-canary", "signing-canary", "client-canary", "bot-canary", "oauth-code-canary", "provider-echo-canary"]; const chunks: string[] = []; const stream = new Writable({ write(chunk, _encoding, callback) { chunks.push(chunk.toString()); callback(); } }); @@ -20,6 +20,7 @@ describe("HTTP logger redaction", () => { app.use(express.json()); app.use(createHttpLogger(pino({ redact: [...HTTP_LOG_REDACT_PATHS] }, stream))); app.use((req, res, next) => { + if (req.path.includes("observability")) res.setHeader("x-provider-echo", canaries[0]); if (status === 500) { next(new Error(canaries.join(" "))); return; } res.status(status).json({ ok: status === 200 }); }); @@ -31,6 +32,14 @@ describe("HTTP logger redaction", () => { })); } responses.push(await request(app).get("/api/chat-slack/oauth/callback").query({ code: canaries[4], error_description: canaries[5] })); + responses.push(await request(app).put("/api/companies/company/observability").query({ unexpected: canaries[2] }).set("x-diagnostic", canaries[3]).send({ + credentials: { mode: "replace", apiKey: canaries[0] }, + unexpected: canaries[1], + })); + responses.push(await request(app).put("/api/companies/company/observability").send({ + credentials: { mode: "replace", headers: [{ name: "x-custom", value: canaries[1] }] }, + unexpected: canaries[0], + })); for (const response of responses) expect(response.status).toBe(status); const output = JSON.stringify({ logs: chunks, responses: responses.map(response => response.body) }); for (const canary of canaries) expect(output).not.toContain(canary); diff --git a/server/src/__tests__/invite-rate-limit-route.test.ts b/server/src/__tests__/invite-rate-limit-route.test.ts index 894034e8b29..d72e693516d 100644 --- a/server/src/__tests__/invite-rate-limit-route.test.ts +++ b/server/src/__tests__/invite-rate-limit-route.test.ts @@ -37,10 +37,8 @@ function createDbStub(...selectResponses: unknown[][]) { } async function createApp(db: Record) { - const [{ accessRoutes }, { errorHandler }] = await Promise.all([ - import("../routes/access.js"), - import("../middleware/index.js"), - ]); + const { errorHandler } = await import("../middleware/index.js"); + const { accessRoutes } = await import("../routes/access.js"); const app = express(); app.use((req, _res, next) => { (req as any).actor = { type: "anon" }; diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 82791597156..79baae125ee 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -25,6 +25,7 @@ const apiPrefixes: Record = { "announcements.ts": "/api", "ai-connections.ts": "/api", "decision-models.ts": "/api", + "agent-observability.ts": "/api", "fast-responses.ts": "/api", "attention.ts": "/api", "approvals.ts": "/api", @@ -410,6 +411,19 @@ describe("openapi routes", () => { expect(Object.keys(remove.responses)).toEqual(expect.arrayContaining(["200", "400", "401", "403", "404", "409"])); }); + it("documents masked board-only agent export settings and revision-checked writes", () => { + const { spec } = loadSpecRoutes(); + const settings = spec.paths["/api/companies/{companyId}/observability"]; + const test = spec.paths["/api/companies/{companyId}/observability/test"].post; + for (const operation of [settings.get, settings.put, settings.delete, test]) { + expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]); + } + expect(settings.delete.requestBody.content["application/json"].schema).toMatchObject({ required: ["expectedRevision"], additionalProperties: false }); + expect(test.responses["409"]).toBeDefined(); + expect(JSON.stringify(settings.get.responses)).not.toMatch(/credentialsSecretId|ciphertext|apiKey/); + expect(test.responses["200"].content["application/json"].schema.properties.source).toMatchObject({ enum: ["control_plane"] }); + }); + it("documents personal board-only announcements and private responses", () => { const { spec } = loadSpecRoutes(); const current = spec.paths["/api/announcements/current"].get; diff --git a/server/src/__tests__/routines-e2e.test.ts b/server/src/__tests__/routines-e2e.test.ts index d08c2b51ada..d50ce624187 100644 --- a/server/src/__tests__/routines-e2e.test.ts +++ b/server/src/__tests__/routines-e2e.test.ts @@ -161,10 +161,8 @@ describeEmbeddedPostgres("routine routes end-to-end", () => { }); async function createApp(actor: Record) { - const [{ routineRoutes }, { errorHandler }] = await Promise.all([ - import("../routes/routines.js"), - import("../middleware/index.js"), - ]); + const { errorHandler } = await import("../middleware/index.js"); + const { routineRoutes } = await import("../routes/routines.js"); const app = express(); app.use(express.json()); app.use((req, _res, next) => { diff --git a/server/src/__tests__/server-package-otel-peer-metadata.test.ts b/server/src/__tests__/server-package-otel-peer-metadata.test.ts index 1473622d0b6..b2880ebd1d2 100644 --- a/server/src/__tests__/server-package-otel-peer-metadata.test.ts +++ b/server/src/__tests__/server-package-otel-peer-metadata.test.ts @@ -3,17 +3,16 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; /** - * The seven OpenTelemetry packages `server/src/instrumentation.ts` loads - * through a dynamic import. Each one must be an optional peer dependency, not - * a normal dependency, so a default install never pulls in the SDK. This test - * guards the manifest half of that contract; `instrumentation.test.ts` guards - * the runtime half (the bootstrap fails open when a package is absent). + * Infrastructure tracing providers and exporters remain optional peers. + * Company connection tests use the resource utility and OTLP serializers as + * regular dependencies, without creating a provider or enabling tracing. + * `instrumentation.test.ts` guards opt-in provider initialization and fail-open + * bootstrap when the optional provider packages are absent. */ const OPTIONAL_OTEL_PACKAGES = [ "@opentelemetry/sdk-node", "@opentelemetry/auto-instrumentations-node", - "@opentelemetry/resources", "@opentelemetry/semantic-conventions", "@opentelemetry/exporter-trace-otlp-grpc", "@opentelemetry/exporter-trace-otlp-proto", @@ -50,4 +49,11 @@ describe("server package OpenTelemetry peer metadata", () => { expect(packageJson.dependencies?.["@opentelemetry/api"]).toBeDefined(); expect(packageJson.peerDependencies?.["@opentelemetry/api"]).toBeUndefined(); }); + + it.each(["@opentelemetry/resources", "@opentelemetry/otlp-transformer"])( + "provides %s for explicit company connection tests", + (packageName) => { + expect(packageJson.dependencies?.[packageName]).toBeDefined(); + }, + ); }); diff --git a/server/src/__tests__/sidebar-preferences-routes.test.ts b/server/src/__tests__/sidebar-preferences-routes.test.ts index f426b30509f..3bdb2d72b21 100644 --- a/server/src/__tests__/sidebar-preferences-routes.test.ts +++ b/server/src/__tests__/sidebar-preferences-routes.test.ts @@ -18,10 +18,8 @@ function registerModuleMocks() { } async function createApp(actor: Record) { - const [{ sidebarPreferenceRoutes }, { errorHandler }] = await Promise.all([ - import("../routes/sidebar-preferences.js"), - import("../middleware/index.js"), - ]); + const { errorHandler } = await import("../middleware/index.js"); + const { sidebarPreferenceRoutes } = await import("../routes/sidebar-preferences.js"); const app = express(); app.use(express.json()); app.use((req, _res, next) => { diff --git a/server/src/app.ts b/server/src/app.ts index 9a35a999a52..841046cd992 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -34,6 +34,7 @@ import { createPublicMcpEvents, type PublicMcpEvents } from "./services/public-m import { publicMcpIngressRoutes, publicMcpManagementRoutes } from "./routes/public-mcp.js"; import { agentAvatarRoutes } from "./routes/agent-avatars.js"; import { decisionModelRoutes } from "./routes/decision-models.js"; +import { agentObservabilityRoutes } from "./routes/agent-observability.js"; import { aiConnectionRoutes } from "./routes/ai-connections.js"; import { taskWorkspaceRoutes } from "./routes/task-workspaces.js"; import { projectToolRoutes } from "./routes/project-tools.js"; @@ -951,6 +952,7 @@ export async function createApp( app.locals.toolActionDeliveries = toolActionDeliveries; app.use(mcpGatewayProtocolRoutes(toolGateway)); api.use(decisionModelRoutes(db)); + api.use(agentObservabilityRoutes(db, { deploymentMode: opts.deploymentMode, deploymentExposure: opts.deploymentExposure })); api.use(fastResponseRoutes(db)); api.use(aiConnectionRoutes(db, { deploymentMode: opts.deploymentMode, deploymentExposure: opts.deploymentExposure, trustedLocalStdioRuntimeHost })); api.use( diff --git a/server/src/middleware/http-log-policy.ts b/server/src/middleware/http-log-policy.ts index 5d4a975fd85..69f0b3cb945 100644 --- a/server/src/middleware/http-log-policy.ts +++ b/server/src/middleware/http-log-policy.ts @@ -54,6 +54,13 @@ export function isPrivateAgentCommentaryHttpRequest(url: string | undefined): bo return /^\/api\/companies\/[^/]+\/agent-commentary(?:\/|$)/i.test(pathname); } +/** Destination credentials stay private even on malformed settings requests. */ +export function isPrivateAgentObservabilityHttpRequest(url: string | undefined): boolean { + if (!url) return false; + const pathname = normalizePath(url).replace(/^https?:\/\/[^/]*/i, ""); + return /^\/api\/companies\/[^/]+\/observability(?:\/|$)/i.test(pathname); +} + /** Muse OAuth and worker payloads stay private even on malformed/rejected paths. */ export function isPrivateMuseHttpRequest(url:string|undefined):boolean { if(!url)return false; @@ -102,6 +109,9 @@ export function isSecretSensitiveHttpRequest( if (isPrivateWebhookHttpRequest(method, url) || isPrivateMuseHttpRequest(url)) return true; if (url && normalizePath(url).startsWith("/mcp/files/")) return true; if (isPrivateAgentCommentaryHttpRequest(url)) return true; + // Custom OTLP header values are arbitrary credentials, including on rejected + // methods or malformed subpaths. Named-field redaction cannot identify them. + if (isPrivateAgentObservabilityHttpRequest(url)) return true; // Avatar uploads and dedicated Dot MCP calls may contain private image bytes. if (url && /^(?:\/mcp\/runner(?:\/|$)|\/api\/companies\/[^/]+\/agents\/[^/]+\/avatar(?:\/|$))/i.test(normalizePath(url).replace(/^https?:\/\/[^/]*/i, ""))) return true; if (!method || !url) return false; diff --git a/server/src/middleware/logger.ts b/server/src/middleware/logger.ts index 093c316ff1d..6ca42fa3532 100644 --- a/server/src/middleware/logger.ts +++ b/server/src/middleware/logger.ts @@ -6,6 +6,7 @@ import { isPrivateWebhookHttpRequest, isSecretSensitiveHttpRequest, isPrivateAgentCommentaryHttpRequest, + isPrivateAgentObservabilityHttpRequest, isPrivateMuseHttpRequest, shouldSilenceHttpSuccessLog, } from "./http-log-policy.js"; @@ -76,6 +77,7 @@ function requestLogUrl(req: { url?: unknown; }) { if(isPrivateMuseHttpRequest(requestClassificationUrl(req)))return "/api/muse/:protocol"; + if (isPrivateAgentObservabilityHttpRequest(requestClassificationUrl(req))) return "/api/companies/:companyId/observability"; return isPrivateWebhook(req) ? privateWebhookLogUrl(requestClassificationUrl(req)) : stripSecretBearingUrlParts(typeof req.url === "string" ? req.url : ""); @@ -86,6 +88,9 @@ export function createHttpLogger(baseLogger: Logger) { logger: baseLogger, serializers: { req(req: Record & { url?: unknown }) { + if (isPrivateAgentObservabilityHttpRequest(typeof req.url === "string" ? req.url : undefined)) { + return { id: req.id, method: req.method, url: "/api/companies/:companyId/observability" }; + } if(isPrivateMuseHttpRequest(typeof req.url === "string" ? req.url : undefined))return {id:req.id,method:req.method,url:"/api/muse/:protocol"}; if (isPrivateAgentCommentaryHttpRequest(typeof req.url === "string" ? req.url : undefined)) { return { id: req.id, method: req.method, url: stripSecretBearingUrlParts(req.url as string) }; @@ -126,7 +131,7 @@ export function createHttpLogger(baseLogger: Logger) { ) { // A provider error may also be reflected in response headers. Keep the // same content-free contract on both sides of a webhook request. - return res.raw?.req && (isPrivateWebhook(res.raw.req) || isPrivateMuseHttpRequest(requestClassificationUrl(res.raw.req)) || isPrivateAgentCommentaryHttpRequest(requestClassificationUrl(res.raw.req))) + return res.raw?.req && (isPrivateWebhook(res.raw.req) || isPrivateMuseHttpRequest(requestClassificationUrl(res.raw.req)) || isPrivateAgentCommentaryHttpRequest(requestClassificationUrl(res.raw.req)) || isPrivateAgentObservabilityHttpRequest(requestClassificationUrl(res.raw.req))) ? { statusCode: res.statusCode } : res; }, @@ -157,6 +162,9 @@ export function createHttpLogger(baseLogger: Logger) { return `${req.method} ${stripSecretBearingUrlParts(req.url ?? "")} ${res.statusCode} — ${errMsg}`; }, customErrorObject(req, _res, _err, value) { + if (isPrivateAgentObservabilityHttpRequest(requestClassificationUrl(req))) { + return { ...value, err: { type: "Error", message: "Agent observability request failed" } }; + } if (isPrivateMuseHttpRequest(requestClassificationUrl(req)))return {...value,err:{type:"Error",message:"External agent request failed"}}; if (isPrivateAgentCommentaryHttpRequest(requestClassificationUrl(req))) { return { ...value, err: { type: "Error", message: "Feedback request failed" } }; @@ -176,7 +184,7 @@ export function createHttpLogger(baseLogger: Logger) { if (/^\/mcp\/(?:oauth|paperclip)(?:\/|$)/.test(requestClassificationUrl(req) ?? "")) { return { reqBody: "[REDACTED]", ...(ctx ? { errorContext: { name: "Error" } } : {}) }; } - if (isPrivateWebhook(req) || isPrivateMuseHttpRequest(requestClassificationUrl(req)) || isPrivateAgentCommentaryHttpRequest(requestClassificationUrl(req))) { + if (isPrivateWebhook(req) || isPrivateMuseHttpRequest(requestClassificationUrl(req)) || isPrivateAgentCommentaryHttpRequest(requestClassificationUrl(req)) || isPrivateAgentObservabilityHttpRequest(requestClassificationUrl(req))) { // Omit, rather than recursively redact, the entire provider payload. // This applies equally before/after parsing and with/without context. return { diff --git a/server/src/routes/agent-observability.ts b/server/src/routes/agent-observability.ts new file mode 100644 index 00000000000..2e33e5804d7 --- /dev/null +++ b/server/src/routes/agent-observability.ts @@ -0,0 +1,44 @@ +import { Router, type Request } from "express"; +import type { Db } from "@paperclipai/db"; +import { agentObservabilityRevisionSchema, updateAgentObservabilitySchema } from "@paperclipai/shared"; +import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js"; +import { canManageAiConnections } from "./ai-connections.js"; +import { forbidden } from "../errors.js"; +import { validate } from "../middleware/validate.js"; +import { agentObservabilityService } from "../services/agent-observability.js"; +import { testAgentObservability } from "../services/agent-observability-test.js"; + +export function agentObservabilityRoutes(db: Db, options: { deploymentMode?: string; deploymentExposure?: string } = {}) { + const router = Router(), service = agentObservabilityService(db); + const path = "/companies/:companyId/observability"; + async function access(req: Request, manage: boolean) { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); assertBoard(req); + const canManage = await canManageAiConnections(db, req, companyId); + if (manage && !canManage) throw forbidden("Manage connections permission is required"); + return { companyId, canManage, userId: getActorInfo(req).actorId }; + } + router.get("/companies/:companyId/observability", async (req, res) => { + const { companyId, canManage } = await access(req, false); + res.json({ canManage, settings: canManage ? await service.settings(companyId) : null }); + }); + router.use(path, async (req, _res, next) => { await access(req, true); next(); }); + router.put("/companies/:companyId/observability", validate(updateAgentObservabilitySchema), async (req, res) => { + const { companyId, userId } = await access(req, true); + res.json({ canManage: true, settings: await service.configure(companyId, userId, req.body) }); + }); + router.delete("/companies/:companyId/observability", validate(agentObservabilityRevisionSchema), async (req, res) => { + const { companyId, userId } = await access(req, true); + await service.remove(companyId, userId, req.body.expectedRevision); + res.json({ canManage: true, settings: null }); + }); + router.post("/companies/:companyId/observability/test", validate(agentObservabilityRevisionSchema), async (req, res) => { + const { companyId, userId } = await access(req, true); + const revision = req.body.expectedRevision as string; + const resolved = await service.resolve(companyId, { forTest: true, expectedRevision: revision, userId }); + const receipt = await testAgentObservability({ companyId, revision, resolved, + allowPrivateNetwork: options.deploymentMode !== "authenticated" || options.deploymentExposure !== "public" }); + res.json(await service.recordTest(companyId, userId, receipt)); + }); + return router; +} diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 2496ec184a9..e42454f9b48 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -13,6 +13,10 @@ import { toggleAllGitHubRepositoriesSchema, createAiConnectionSchema, updateDecisionModelSchema, + updateAgentObservabilitySchema, + agentObservabilityRevisionSchema, + agentObservabilityCaptureSchema, + agentObservabilityDestinationSchema, aiConnectionPoolConfigSchema, aiConnectionLoginIntentSchema, localAiConnectionSchema, @@ -1389,6 +1393,10 @@ const BOARD_ONLY_OPERATIONS = new Set([ "GET /api/companies/{companyId}/decision-model", "PUT /api/companies/{companyId}/decision-model", "POST /api/companies/{companyId}/decision-model/test", + "GET /api/companies/{companyId}/observability", + "PUT /api/companies/{companyId}/observability", + "DELETE /api/companies/{companyId}/observability", + "POST /api/companies/{companyId}/observability/test", ...browserUseOperations.map(([method, path]) => `${method.toUpperCase()} ${path}`), "GET /api/companies/{companyId}/ai-connections", "POST /api/companies/{companyId}/ai-connections", @@ -11089,6 +11097,41 @@ for (const [method, suffix, summary] of [ ] as const) registerCurrentRoute({ method, path: `/api/companies/{companyId}/fast-response${suffix}`, tags: ["fast-responses"], summary, ...(method === "put" ? { body: updateFastResponseSchema } : {}), responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 422: r.unprocessable } }); +const observabilitySignalReceiptSchema = z.object({ + status: z.enum(["accepted", "rejected", "failed"]), httpStatus: z.number().int().nullable(), + rejectedRecords: z.number().int().nonnegative(), errorCode: z.string().nullable(), message: z.string().nullable(), durationMs: z.number().nonnegative(), +}).strict(); +const observabilityReceiptSchema = z.object({ + revision: z.string().uuid(), marker: z.string(), source: z.literal("control_plane"), + startedAt: z.string().datetime(), finishedAt: z.string().datetime(), status: z.enum(["accepted", "partial", "failed"]), + signals: z.object({ logs: observabilitySignalReceiptSchema, traces: observabilitySignalReceiptSchema }).strict(), +}).strict(); +const observabilitySettingsEnvelopeSchema = z.object({ canManage: z.boolean(), settings: z.object({ + companyId: z.string().uuid(), revision: z.string().uuid(), enabled: z.boolean(), destination: agentObservabilityDestinationSchema, + capture: agentObservabilityCaptureSchema, credentialsConfigured: z.boolean(), headerNames: z.array(z.string()), + updatedAt: z.string().datetime(), lastTest: observabilityReceiptSchema.nullable(), +}).strict().nullable() }).strict(); +registerCurrentRoute({ + method: "get", path: "/api/companies/{companyId}/observability", tags: ["agent-observability"], + summary: "Read masked company agent export settings; board connection managers only", + responses: { 200: r.ok(observabilitySettingsEnvelopeSchema), 401: r.unauthorized, 403: r.forbidden }, +}); +registerCurrentRoute({ + method: "put", path: "/api/companies/{companyId}/observability", tags: ["agent-observability"], + summary: "Save revision-checked agent export settings without contacting the destination", body: updateAgentObservabilitySchema, + responses: { 200: r.ok(observabilitySettingsEnvelopeSchema), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 409: r.conflict, 422: r.unprocessable }, +}); +registerCurrentRoute({ + method: "delete", path: "/api/companies/{companyId}/observability", tags: ["agent-observability"], + summary: "Remove revision-checked company export settings and retire owned credentials", body: agentObservabilityRevisionSchema, + responses: { 200: r.ok(observabilitySettingsEnvelopeSchema), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, +}); +registerCurrentRoute({ + method: "post", path: "/api/companies/{companyId}/observability/test", tags: ["agent-observability"], + summary: "Send synthetic logs and traces from the control plane; destination failure returns a sanitized receipt", body: agentObservabilityRevisionSchema, + responses: { 200: r.ok(observabilityReceiptSchema), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, +}); + registerCurrentRoute({ method: "get", path: "/api/companies/{companyId}/decision-model", tags: ["decision-models"], summary: "Get decision settings, compatible connections, and manager capability", diff --git a/server/src/services/agent-observability-launch.ts b/server/src/services/agent-observability-launch.ts new file mode 100644 index 00000000000..739abbb7892 --- /dev/null +++ b/server/src/services/agent-observability-launch.ts @@ -0,0 +1,9 @@ +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE } from "@paperclipai/shared"; +import { buildProviderObservabilityLaunch } from "@paperclipai/adapter-utils/agent-observability"; +import type { ResolvedAgentObservability } from "./agent-observability.js"; + +export function agentObservabilityLaunch(snapshot: ResolvedAgentObservability, input: { kind: "codex" | "claude"; companyId: string; agentId: string }) { + return buildProviderObservabilityLaunch({ ...input, revision: snapshot.revision, compatibilityKey: snapshot.compatibilityKey, + enabled: snapshot.state === "enabled", capture: snapshot.state === "enabled" ? snapshot.capture : DEFAULT_AGENT_OBSERVABILITY_CAPTURE, + ...(snapshot.state === "enabled" ? { logsEndpoint: snapshot.logsEndpoint, tracesEndpoint: snapshot.tracesEndpoint, headers: snapshot.headers } : {}) }); +} diff --git a/server/src/services/agent-observability-test.ts b/server/src/services/agent-observability-test.ts new file mode 100644 index 00000000000..7f099240fca --- /dev/null +++ b/server/src/services/agent-observability-test.ts @@ -0,0 +1,113 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { resourceFromAttributes } from "@opentelemetry/resources"; +import { ProtobufLogsSerializer, ProtobufTraceSerializer } from "@opentelemetry/otlp-transformer"; +import type { AgentObservabilityTestReceipt, OtlpSignalTestResult } from "@paperclipai/shared"; +import { guardedRemoteHttpFetch } from "./remote-http-fetch.js"; +import type { ResolvedAgentObservability } from "./agent-observability.js"; + +const failureMessages: Record = { + credentials_unavailable: "The saved credentials are unavailable. Replace them and try again.", + configuration_invalid: "The saved configuration is invalid. Save it again.", + timeout: "The receiver did not respond within ten seconds.", + network_error: "The receiver could not be reached from this server.", + receiver_rejected: "The receiver rejected the test records. Check credentials and destination routing.", + partial_rejection: "The receiver reported rejected test records.", + invalid_response: "The receiver returned an invalid OTLP response.", +}; +const failed = (code: string, durationMs = 0, httpStatus: number | null = null): OtlpSignalTestResult => ({ + status: "failed", httpStatus, rejectedRecords: 0, errorCode: code, + message: failureMessages[code] ?? failureMessages.network_error, durationMs, +}); + +async function boundedBody(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const next = await reader.read(); + if (next.done) break; + size += next.value.length; + if (size > 65536) throw new Error("OTLP response too large"); + chunks.push(next.value); + } + } finally { await reader.cancel().catch(() => {}); } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } + return bytes; +} + +export async function testAgentObservability(input: { + companyId: string; revision: string; resolved: ResolvedAgentObservability; + allowPrivateNetwork: boolean; timeoutMs?: number; +}): Promise { + const startedAt = new Date().toISOString(); + const marker = `paperclip-observability-test-${randomUUID()}`; + const traceId = randomBytes(16).toString("hex"), spanId = randomBytes(8).toString("hex"); + const spanContext = { traceId, spanId, traceFlags: 1 }; + const now = Date.now(); + const time: [number, number] = [Math.floor(now / 1000), (now % 1000) * 1_000_000]; + const attributes = { "paperclip.company.id": input.companyId, "paperclip.observability.revision": input.revision, "paperclip.observability.test_marker": marker }; + const resource = resourceFromAttributes({ "service.name": "paperclip-observability-test", ...attributes }); + const instrumentationScope = { name: "paperclip.agent-observability", version: "1" }; + const log: Parameters[0][number] = { + resource, instrumentationScope, hrTime: time, hrTimeObserved: time, + severityNumber: 9, severityText: "INFO", body: marker, attributes, droppedAttributesCount: 0, spanContext, + }; + const span: Parameters[0][number] = { + name: "paperclip.observability.connection_test", kind: 0, spanContext: () => spanContext, + startTime: time, endTime: time, duration: [0, 0], status: { code: 1 }, attributes, + links: [], events: [], ended: true, resource, instrumentationScope, + droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0, + }; + const abort = new AbortController(); + const timeoutMs = input.timeoutMs ?? 10000; + const timer = setTimeout(() => abort.abort(), timeoutMs); + try { + const send = async (signal: "logs" | "traces"): Promise => { + const started = performance.now(); + const duration = () => Math.round(performance.now() - started); + if (input.resolved.state !== "enabled") return failed(input.resolved.reason ?? "configuration_invalid"); + const serializer = signal === "logs" ? ProtobufLogsSerializer : ProtobufTraceSerializer; + const payload = signal === "logs" ? ProtobufLogsSerializer.serializeRequest([log]) : ProtobufTraceSerializer.serializeRequest([span]); + if (!payload) return failed("configuration_invalid"); + let status: number | null = null; + try { + const response = await guardedRemoteHttpFetch(signal === "logs" ? input.resolved.logsEndpoint : input.resolved.tracesEndpoint, { + method: "POST", headers: { ...input.resolved.headers, "content-type": "application/x-protobuf", accept: "application/x-protobuf, application/json" }, + body: Buffer.from(payload), signal: abort.signal, + }, { allowPrivateNetwork: input.allowPrivateNetwork, dnsTimeoutMs: timeoutMs, connectTimeoutMs: timeoutMs, + responseTimeoutMs: timeoutMs, error: () => new Error("Observability endpoint is unavailable") }); + status = response.status; + const bytes = await boundedBody(response); + if (!response.ok) return failed("receiver_rejected", duration(), status); + const contentType = response.headers.get("content-type")?.split(";")[0].trim(); + if (contentType && !["application/x-protobuf", "application/json"].includes(contentType)) return failed("invalid_response", duration(), status); + let rejected = 0; + try { + if (bytes.length) { + let result: unknown; + if (contentType === "application/json") result = JSON.parse(new TextDecoder().decode(bytes)); + else if (contentType === "application/x-protobuf") result = serializer.deserializeResponse(bytes); + else return failed("invalid_response", duration(), status); + if (!result || typeof result !== "object" || Array.isArray(result)) return failed("invalid_response", duration(), status); + const partial = (result as { partialSuccess?: { rejectedLogRecords?: unknown; rejectedSpans?: unknown } }).partialSuccess; + if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) return failed("invalid_response", duration(), status); + const count = partial?.[signal === "logs" ? "rejectedLogRecords" : "rejectedSpans"]; + if (count !== undefined && (typeof count !== "number" && typeof count !== "string" || typeof count === "string" && !/^\d+$/.test(count))) return failed("invalid_response", duration(), status); + rejected = count === undefined ? 0 : Number(count); + if (!Number.isSafeInteger(rejected) || rejected < 0) return failed("invalid_response", duration(), status); + } + } catch { return failed("invalid_response", duration(), status); } + return { status: rejected ? "rejected" : "accepted", httpStatus: status, rejectedRecords: rejected, + errorCode: rejected ? "partial_rejection" : null, message: rejected ? failureMessages.partial_rejection : null, durationMs: duration() }; + } catch { return failed(abort.signal.aborted ? "timeout" : "network_error", duration(), status); } + }; + const [logs, traces] = await Promise.all([send("logs"), send("traces")]); + const accepted = [logs, traces].filter(result => result.status === "accepted").length; + return { revision: input.revision, marker, source: "control_plane", startedAt, finishedAt: new Date().toISOString(), + status: accepted === 2 ? "accepted" : accepted || [logs, traces].some(result => result.status === "rejected") ? "partial" : "failed", signals: { logs, traces } }; + } finally { clearTimeout(timer); } +} diff --git a/server/src/services/agent-observability.ts b/server/src/services/agent-observability.ts new file mode 100644 index 00000000000..ad1e9328ae0 --- /dev/null +++ b/server/src/services/agent-observability.ts @@ -0,0 +1,182 @@ +import { createHash, randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { companyObservability, type Db } from "@paperclipai/db"; +import { + agentObservabilityCaptureSchema, agentObservabilityDestinationSchema, agentObservabilityHeadersSchema, + DEFAULT_AGENT_OBSERVABILITY_CAPTURE, updateAgentObservabilitySchema, + isSameOtlpOrigin, + type AgentObservabilitySettings, type AgentObservabilityTestReceipt, type UpdateAgentObservabilityRequest, type AgentObservabilityUnavailableReason, +} from "@paperclipai/shared"; +import { conflict, notFound, unprocessable } from "../errors.js"; +import { withAccountingTransaction } from "./accounting-transaction.js"; +import { logActivity } from "./activity-log.js"; +import { secretService } from "./secrets.js"; +import { withAccountHomeSecretMutationLock } from "@paperclipai/adapter-codex-local/server"; + +type Row = typeof companyObservability.$inferSelect; +const target = { targetType: "system" as const, targetId: "company-observability" }; +const binding = { consumerType: "system" as const, consumerId: target.targetId, configPath: "headers", actorType: "system" as const }; +const credentialSchema = "paperclip.agent_observability.credentials.v1"; +type ResolutionAccess = { runId?: string; agentId?: string; userId?: string; responsibleUserId?: string | null; issueId?: string | null }; +export type ResolvedAgentObservability = { + state: "disabled" | "unavailable"; revision: string | null; compatibilityKey: string; reason?: string; +} | { + state: "enabled"; revision: string; compatibilityKey: string; + logsEndpoint: string; tracesEndpoint: string; headers: Readonly>; + capture: typeof DEFAULT_AGENT_OBSERVABILITY_CAPTURE | AgentObservabilitySettings["capture"]; +}; + +function identity(companyId: string, revision: string | null, state: string, version?: number) { + return createHash("sha256").update(JSON.stringify(["paperclip.agent-observability.launch.v1", companyId, revision, state, version ?? null])).digest("hex"); +} +export function unavailableAgentObservability(companyId: string, reason: AgentObservabilityUnavailableReason, revision: string | null = null): ResolvedAgentObservability { + return { state: "unavailable", reason, revision, compatibilityKey: identity(companyId, revision, `unavailable:${reason}`) }; +} +function settingsDto(row: Row): AgentObservabilitySettings { + return { companyId: row.companyId, revision: row.revision, enabled: row.enabled, + destination: agentObservabilityDestinationSchema.parse(row.destination), capture: agentObservabilityCaptureSchema.parse(row.capture), + credentialsConfigured: Boolean(row.credentialsSecretId), headerNames: row.headerNames, + updatedAt: row.updatedAt.toISOString(), lastTest: row.lastTest }; +} +function assertRevision(row: Row | undefined, expectedRevision: string | null) { + if ((row?.revision ?? null) !== expectedRevision) { + throw conflict("Agent observability settings changed. Reload saved settings before saving again.", { code: "observability_revision_conflict" }); + } +} + +export function agentObservabilityService(db: Db) { + const getRow = (companyId: string, executor = db) => executor.select().from(companyObservability) + .where(eq(companyObservability.companyId, companyId)).then(rows => rows[0]); + + async function readCredentials(companyId: string, row: Row, executor = db, access: ResolutionAccess = {}): Promise<{ headers: Record; version?: number }> { + if (!row.credentialsSecretId) { + if (row.destination.provider === "braintrust" || row.headerNames.length) throw unprocessable("Observability credentials are unavailable"); + return { headers: {} }; + } + const secrets = secretService(executor); + const context = { ...binding, heartbeatRunId: access.runId ?? null, issueId: access.issueId ?? null, + responsibleUserId: access.responsibleUserId ?? access.userId ?? null, + actorType: access.userId ? "user" as const : access.agentId ? "agent" as const : "system" as const, + actorId: access.userId ?? access.agentId ?? null }; + const version = await secrets.resolveSecretVersion(companyId, row.credentialsSecretId, "latest", context); + const raw = await secrets.resolveSecretValue(companyId, row.credentialsSecretId, version, context); + const value: unknown = JSON.parse(raw); + if (!value || typeof value !== "object" || !("schema" in value) || value.schema !== credentialSchema || !("headers" in value) + || !value.headers || typeof value.headers !== "object" || Array.isArray(value.headers)) throw unprocessable("Observability credentials are unavailable"); + const entries = agentObservabilityHeadersSchema.parse(Object.entries(value.headers).map(([name, header]) => ({ name, value: header }))); + const headers = Object.fromEntries(entries.map(entry => [entry.name, entry.value!])); + if (row.destination.provider === "braintrust" && !headers.authorization) throw unprocessable("Observability credentials are unavailable"); + return { headers, version }; + } + + return { + settings: async (companyId: string) => { const row = await getRow(companyId); return row ? settingsDto(row) : null; }, + configure: async (companyId: string, userId: string, input: UpdateAgentObservabilityRequest) => { + const data = updateAgentObservabilitySchema.parse(input); + // Match credential writers: filesystem mutation ownership precedes DB row locks. + return withAccountHomeSecretMutationLock(undefined, companyId, () => withAccountingTransaction(db, companyId, async (tx, publications) => { + const previous = await getRow(companyId, tx); + assertRevision(previous, data.expectedRevision); + if ((!previous || previous.destination.provider !== data.destination.provider) && data.credentials.mode === "keep") { + throw unprocessable("Provide credentials when choosing a new destination", { code: "observability_credentials_required" }); + } + const originChanged = previous?.destination.provider === "otlp" && data.destination.provider === "otlp" + && !isSameOtlpOrigin(previous.destination, data.destination); + if (originChanged && (data.credentials.mode === "keep" && previous.headerNames.length > 0 + || data.credentials.mode === "replace" && "headers" in data.credentials && data.credentials.headers.some(header => header.value === undefined))) { + throw unprocessable("Re-enter every header value for the new endpoint, or remove the header", { code: "observability_credentials_required" }); + } + const secrets = secretService(tx); + let secretId = previous?.credentialsSecretId ?? null; + let headerNames = previous?.headerNames ?? []; + if (data.credentials.mode === "replace") { + let headers: Record; + if ("apiKey" in data.credentials) { + headers = { authorization: `Bearer ${data.credentials.apiKey}` }; + } else { + const needsSaved = data.credentials.headers.some(header => header.value === undefined); + const saved = needsSaved && previous && previous.destination.provider === "otlp" + ? (await readCredentials(companyId, previous, tx, { userId })).headers : {}; + headers = Object.fromEntries(data.credentials.headers.map(header => { + const value = header.value ?? saved[header.name]; + if (value === undefined) throw unprocessable("New or renamed headers require a value", { code: "observability_header_value_required" }); + return [header.name, value]; + })); + } + agentObservabilityHeadersSchema.parse(Object.entries(headers).map(([name, value]) => ({ name, value }))); + headerNames = Object.keys(headers).sort(); + if (headerNames.length) { + const value = JSON.stringify({ schema: credentialSchema, headers }); + if (secretId) { + // Existing inactive/deleted credentials may be replaced without resolving them. + const old = await secrets.getById(secretId); + if (old?.status === "active") await secrets.rotate(secretId, { value }); + else secretId = null; + } + if (!secretId) { + const suffix = randomUUID(); + const secret = await secrets.create(companyId, { name: `agent-observability.${suffix}`, key: `AGENT_OBSERVABILITY_HEADERS_${suffix.replaceAll("-", "_")}`, + provider: "local_encrypted", managedMode: "paperclip_managed", value }, { userId }); + secretId = secret.id; + } + } else secretId = null; + await secrets.syncSecretRefsForTarget(companyId, target, secretId ? [{ secretId, configPath: "headers", versionSelector: "latest", projectionClass: "unclassified" }] : [], { replaceAll: true }); + } + if (data.enabled && data.destination.provider === "braintrust" && !secretId) { + throw unprocessable("Provide a Braintrust API key", { code: "observability_credentials_required" }); + } + const values = { companyId, revision: randomUUID(), enabled: data.enabled, destination: data.destination, + capture: data.capture, credentialsSecretId: secretId, headerNames, lastTest: null, updatedAt: new Date() }; + const [row] = await tx.insert(companyObservability).values(values).onConflictDoUpdate({ target: companyObservability.companyId, set: values }).returning(); + if (previous?.credentialsSecretId && previous.credentialsSecretId !== secretId) await secrets.remove(previous.credentialsSecretId); + await logActivity(tx, { companyId, actorType: "user", actorId: userId, action: "agent_observability.configured", entityType: "company", entityId: companyId, + details: { revision: row.revision, enabled: row.enabled, provider: row.destination.provider, capture: row.capture, headerCount: headerNames.length } }, publications); + return settingsDto(row); + })); + }, + remove: async (companyId: string, userId: string, expectedRevision: string) => withAccountHomeSecretMutationLock(undefined, companyId, () => withAccountingTransaction(db, companyId, async (tx, publications) => { + const row = await getRow(companyId, tx); + assertRevision(row, expectedRevision); + if (!row) throw notFound("Agent observability is not configured"); + const secrets = secretService(tx); + await secrets.syncSecretRefsForTarget(companyId, target, [], { replaceAll: true }); + await tx.delete(companyObservability).where(eq(companyObservability.companyId, companyId)); + if (row.credentialsSecretId) await secrets.remove(row.credentialsSecretId); + await logActivity(tx, { companyId, actorType: "user", actorId: userId, action: "agent_observability.removed", entityType: "company", entityId: companyId, + details: { revision: row.revision } }, publications); + })), + resolve: async (companyId: string, options: ResolutionAccess & { forTest?: boolean; expectedRevision?: string; unavailableReason?: AgentObservabilityUnavailableReason } = {}): Promise => { + const row = await getRow(companyId); + if (options.expectedRevision) { if (!row) throw notFound("Agent observability is not configured"); assertRevision(row, options.expectedRevision); } + if (!row || (!row.enabled && !options.forTest)) return { state: "disabled", revision: row?.revision ?? null, compatibilityKey: identity(companyId, row?.revision ?? null, "disabled") }; + const unavailable = (reason: AgentObservabilityUnavailableReason) => unavailableAgentObservability(companyId, reason, row.revision); + if (options.unavailableReason) return unavailable(options.unavailableReason); + let destination: AgentObservabilitySettings["destination"], capture: AgentObservabilitySettings["capture"]; + try { destination = agentObservabilityDestinationSchema.parse(row.destination); capture = agentObservabilityCaptureSchema.parse(row.capture); } + catch { return unavailable("configuration_invalid"); } + try { + const { headers, version } = await readCredentials(companyId, row, db, options); + const base = destination.provider === "braintrust" ? "https://api.braintrust.dev/otel" : destination.endpointBaseUrl; + return { state: "enabled", revision: row.revision, compatibilityKey: identity(companyId, row.revision, "enabled", version), capture, + logsEndpoint: `${base}/v1/logs`, tracesEndpoint: `${base}/v1/traces`, + headers: Object.freeze({ ...headers, ...(destination.provider === "braintrust" ? { "x-bt-parent": `project_id:${destination.projectId}` } : {}) }) }; + } catch { return unavailable("credentials_unavailable"); } + }, + // The company deletion owner acquires mutation ownership before its company lock, and owns commit/publication. + removeForCompanyDeletion: async (companyId: string) => { + const row = await getRow(companyId); + if (!row) return; + const secrets = secretService(db); + await secrets.syncSecretRefsForTarget(companyId, target, [], { replaceAll: true }); + await db.delete(companyObservability).where(eq(companyObservability.companyId, companyId)); + if (row.credentialsSecretId) await secrets.remove(row.credentialsSecretId); + }, + recordTest: async (companyId: string, userId: string, receipt: AgentObservabilityTestReceipt) => withAccountingTransaction(db, companyId, async (tx, publications) => { + assertRevision(await getRow(companyId, tx), receipt.revision); + await tx.update(companyObservability).set({ lastTest: receipt }).where(and(eq(companyObservability.companyId, companyId), eq(companyObservability.revision, receipt.revision))); + await logActivity(tx, { companyId, actorType: "user", actorId: userId, action: "agent_observability.tested", entityType: "company", entityId: companyId, + details: { revision: receipt.revision, status: receipt.status, marker: receipt.marker } }, publications); + return receipt; + }), + }; +} diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index b668c7ad01e..5449677e8de 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,5 @@ +import { agentObservabilityService } from "./agent-observability.js"; +import { withAccountHomeSecretMutationLock } from "@paperclipai/adapter-codex-local/server"; import type { Db } from "@paperclipai/db"; import { eq, inArray } from "drizzle-orm"; import { createAgentLifecycle, scheduleAgentLifecycle } from "./agent-lifecycle.js"; @@ -48,13 +50,14 @@ export async function deleteCompany(db: Db, id: string) { await lifecycle.terminateAgent(agent.id); await scheduleAgentLifecycle(db, agent.id); } - return db.transaction(async (tx) => { + return withAccountHomeSecretMutationLock(undefined, id, () => db.transaction(async (tx) => { // Exclude accounting writers before taking child locks. KEY SHARE must // remain compatible: native writers can already hold a child row while // saving a company-scoped result. FOR UPDATE would deadlock that save. const [existing] = await tx.select({ id: companies.id }).from(companies) .where(eq(companies.id, id)).for("no key update"); if (!existing) return null; + await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id); // Finance can reference costs and both can reference runs. Incidents // reference policies and approvals; delete these dependents first. await tx.delete(financeEvents).where(eq(financeEvents.companyId, id)); @@ -108,5 +111,5 @@ export async function deleteCompany(db: Db, id: string) { .where(eq(companies.id, id)) .returning(); return rows[0] ?? null; - }); + })); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 55d6d15d59a..6472c6b8612 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2593,7 +2593,7 @@ export function heartbeatService( policy: { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory }, environment: { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput }, services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral }, - onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = readSecrets; }, + onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; }, onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; }, onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; }, })); @@ -2601,7 +2601,7 @@ export function heartbeatService( const { mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub, resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, - runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, + runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability, } = executionConfiguration; let { runtimeConfig } = executionConfiguration; const resolvedFailureSecrets = readFailureReportSecrets(); @@ -2700,6 +2700,7 @@ export function heartbeatService( }, })); const preparedInstructions = instructionWorkingCopy = createHeartbeatInstructionWorkingCopy({ + observabilityCompatibilityKey: observability?.compatibilityKey, db, instructionCopies, readRun: () => run!, agent, context, session: { taskSession, taskSessionForRun, sessionConfigMetadata }, workspace: { executionTarget, executionWorkspace, persistedExecutionWorkspace }, @@ -3325,6 +3326,7 @@ export function heartbeatService( configuredTaskEnvironment, adapterEnv, agentIdentity, + observability, managedAiRuntime, runtimeEnv, useHostGitHub, diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index f10acfda751..8597bdbf660 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -1,3 +1,7 @@ +import { agentHarnessType } from "@paperclipai/shared"; +import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability"; +import { agentObservabilityService, unavailableAgentObservability, type ResolvedAgentObservability } from "../agent-observability.js"; +import { agentObservabilityLaunch } from "../agent-observability-launch.js"; import { withSkillActor } from "../skill-access.js"; import type { AuthorizationActor } from "../authorization.js"; import { eq, sql } from "drizzle-orm"; @@ -90,6 +94,19 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear homeMode: bindHeartbeatRunnerHomeMode(parseObject(agent.adapterConfig), persistedNativeExecutionInput).homeMode } : input.config.sourceWorkspaceConfig; const preconfiguredHome = agent.adapterType === "paperclip_runner" && sourceWorkspaceConfig.homeMode === "preconfigured"; + const initialHarness = agentHarnessType(agent.adapterType, sourceWorkspaceConfig); + const unsupportedObservability = !["codex_local", "claude_local"].includes(initialHarness) + ? "unsupported_harness" as const : preconfiguredHome && initialHarness === "codex_local" ? "unsupported_home_mode" as const : undefined; + const observabilitySnapshot: ResolvedAgentObservability = await agentObservabilityService(db).resolve(agent.companyId, { runId: run.id, agentId: agent.id, responsibleUserId, issueId, unavailableReason: unsupportedObservability }) + .catch(() => unavailableAgentObservability(agent.companyId, "configuration_invalid")); + const identityRedactor = createAgentIdentityRedactor(); + const earlyObservability = unsupportedObservability === undefined && (initialHarness === "codex_local" || initialHarness === "claude_local") + ? agentObservabilityLaunch(observabilitySnapshot, { kind: initialHarness === "codex_local" ? "codex" : "claude", companyId: agent.companyId, agentId: agent.id }) : undefined; + identityRedactor.values.push(...observabilitySecretValues(earlyObservability)); + onIdentityRedactorPrepared(identityRedactor); + // The executor keeps these values in its failure redactor before any later + // preparation step can fail. Subsequent provider/identity values extend it. + onFailureSecretsResolved(() => []); const aiBinding = preconfiguredHome ? undefined : input.config.aiBinding; const { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory } = input.policy; const { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral } = input.services; @@ -300,13 +317,30 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined) ? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id)) : undefined; - const identityRedactor = createAgentIdentityRedactor(agentIdentity?.privateKeyPem); + identityRedactor.values.push(...createAgentIdentityRedactor(agentIdentity?.privateKeyPem).values); + const harness = agentHarnessType(agent.adapterType, runtimeConfig); + const observability = unsupportedObservability !== "unsupported_home_mode" && (harness === "codex_local" || harness === "claude_local") + ? agentObservabilityLaunch(observabilitySnapshot, { kind: harness === "codex_local" ? "codex" : "claude", companyId: agent.companyId, agentId: agent.id }) : undefined; + identityRedactor.values.push(...observabilitySecretValues(observability)); + identityRedactor.values.sort((a, b) => b.length - a.length); + if (observabilitySnapshot.state === "unavailable") { + logger.warn({ runId: run.id, reason: observabilitySnapshot.reason }, "Agent observability unavailable; execution continues without export"); + const reason = observabilitySnapshot.reason; + await appendRunEvent(run, { eventType: "agent_observability.unavailable", stream: "system", level: "warn", + message: reason === "unsupported_home_mode" + ? "Company observability was not applied: automatic configuration of caller-owned Codex homes is unsupported." + : reason === "unsupported_harness" ? "Company observability was not applied: this harness is not supported." + : "Company observability is unavailable. Execution continues without company export.", + payload: { reason, revision: observabilitySnapshot.revision } }).catch(() => { + logger.warn({ runId: run.id }, "Could not record the agent observability availability diagnostic"); + }); + } onIdentityRedactorPrepared(identityRedactor); if (agentIdentity) secretKeys.add("PAPERCLIP_AGENT_PRIVATE_KEY"); return { prepared: true as const, mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub, resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, - runtimeSkillEntries, nativeRunnerPreparationSpans, runtimeConfig, agentIdentity, + runtimeSkillEntries, nativeRunnerPreparationSpans, runtimeConfig, agentIdentity, observability, }; } diff --git a/server/src/services/heartbeat/instruction-working-copy.ts b/server/src/services/heartbeat/instruction-working-copy.ts index 1d96c850481..518f42f7aad 100644 --- a/server/src/services/heartbeat/instruction-working-copy.ts +++ b/server/src/services/heartbeat/instruction-working-copy.ts @@ -53,6 +53,7 @@ export function createHeartbeatInstructionCopyRelease( } interface InstructionWorkingCopyInput { + observabilityCompatibilityKey?: string; db: Db; instructionCopies: Copies; // The executor replaces its run row before preparation. Callbacks must see it. @@ -144,11 +145,13 @@ export function createHeartbeatInstructionWorkingCopy(input: InstructionWorkingC if (warmFiles && taskSessionForRun?.lastRunId) { nativeInstructionReservation = await reserveWarmNativeInstructionDirectory({ companyId: agent.companyId, agentId: agent.id, previousRunId: taskSessionForRun.lastRunId, runId: readRun().id, target: executionTarget, + observabilityCompatibilityKey: input.observabilityCompatibilityKey, canReuse: () => instructionCopies.canReuseWarm(agent.companyId, agent.id, taskSessionForRun!.lastRunId!), }); } if (!warmFiles && nativeRuntimeResolution.kind === "native" && priorWorkingCopy.kind === "agent_files" && taskSession) { releaseWarmInstructionPreparation = await claimWarmNativeInstructionCopy({ + observabilityCompatibilityKey: input.observabilityCompatibilityKey, priorExecution: parseNativeExecutionInput(priorFileInput), companyId: agent.companyId, agentId: agent.id, executionWorkspaceId: persistedExecutionWorkspace?.id ?? readRun().id, workspace: executionWorkspace, environmentId: executionTarget?.environmentId ?? null, runId: readRun().id, preparationKey: instructionPreparationKey, forceRetirement: !taskSessionForRun, diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index c860d2a8e28..067b7fa1198 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -133,6 +133,7 @@ export interface HeartbeatRuntimeExecutionInput { configuredTaskEnvironment: Parameters[0]; adapterEnv: Parameters[0]; agentIdentity: AdapterExecutionContext["agentIdentity"]; + observability?: AdapterExecutionContext["observability"]; runtimeEnv: Selection["runtimeEnv"]; useHostGitHub: boolean; githubSelection: { configured: boolean }; @@ -227,7 +228,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task; const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime; const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace; - const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config; + const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, observability, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config; const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session; const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions; const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace; @@ -575,6 +576,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe return executePaperclipNativeSession({ db, execution: nativeExecution, + observability, getFreshSessionHandoff: getNativeFreshSessionHandoff, refreshTools: context.refreshTools === true, conversationMode: isConversation(issueContext), @@ -796,6 +798,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({ getFreshSessionHandoff, agentIdentity, + observability, runId: run.id, agent, runtime: runtimeForAdapter, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 02a2211ee2a..89f2de91d3e 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -7938,6 +7938,26 @@ describe("native warm session supervision", () => { return { base, next, run }; } + it.each([false, true])("compares export process compatibility without changing conversation identity (changed: %s)", async changed => { + const { base } = fixture(`observability-${changed}`); + const next = { ...base, binding: { ...base.binding, runId: `observability-${changed}-next` } }; + const close = vi.fn(async () => undefined); + const shared = { close }; + const observability = (key: string) => ({ kind: "codex" as const, compatibilityKey: key, otelConfig: { exporter: "none", trace_exporter: "none", metrics_exporter: "none" }, environment: {} }); + state.execute.mockReset().mockImplementationOnce(async options => { + options.onSession?.(shared); return result; + }).mockImplementationOnce(async options => { + expect(options.input.session.normalizedSessionId).toBe(base.session.normalizedSessionId); + expect(options.existingSession).toBe(changed ? undefined : shared); + if (changed) expect(close).toHaveBeenCalledWith({ reason: "warm native session configuration changed" }); + else expect(close).not.toHaveBeenCalled(); + return result; + }); + await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", observability: observability("revision-one") }); + await executePaperclipNativeSession({ db: leaseDb(next), execution: next, runnerInstanceId: "runner", observability: observability(changed ? "revision-two" : "revision-one") }); + expect(state.execute).toHaveBeenCalledTimes(2); + }); + it("replaces an older warm process before the first turn with cryptographic identity", async () => { const { base } = fixture("identity-key-upgrade"); const next = { ...base, binding: { ...base.binding, runId: "identity-key-next" } }; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b22a1d7c9ff..4947db3a769 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { applyObservabilityEnvironment, type ProviderObservabilityLaunch } from "@paperclipai/adapter-utils/agent-observability"; import { museRunnerBroker } from "../muse-runner-broker.js"; import { stageRemoteLaunchAssets } from "./remote-launch-assets.js"; import { createRunPreparationTiming } from "../execution-preparation-timing.js"; @@ -441,6 +442,7 @@ export type NativeInstructionWorkingCopy = { }; type WarmNativeSession = { + observabilityCompatibilityKey?: string; runId: string; instructionWorkingCopy?: NativeInstructionWorkingCopy; // Keep the admitted physical root independent of the per-run collection @@ -492,6 +494,7 @@ const closingWarmNativeSessions = new Map>(); /** Reserve the existing physical copy before heartbeat composes AGENT_HOME and * task guidance. The final execution/configuration fence still owns admission. */ export async function claimWarmNativeInstructionCopy(input: { + observabilityCompatibilityKey?: string; priorExecution: NativeExecutionInput; companyId: string; agentId: string; @@ -524,7 +527,7 @@ export async function claimWarmNativeInstructionCopy(input: { } finally { entry.busy = false; } }; try { - const preparationChanged = input.forceRetirement === true || entry.githubAccess?.ready === false || entry.instructionWorkingCopy.preparationKey !== input.preparationKey; + const preparationChanged = input.forceRetirement === true || entry.observabilityCompatibilityKey !== input.observabilityCompatibilityKey || entry.githubAccess?.ready === false || entry.instructionWorkingCopy.preparationKey !== input.preparationKey; // The prior capability may close over an expired remote lease. Rebind // collection authority first; adoption itself grants no reuse permission. const adopted = await input.adopt(entry.instructionWorkingCopy.runId); @@ -559,6 +562,7 @@ function instructionTargetIdentity(target?: AdapterExecutionTarget | null): stri * A persisted conversation alone is not proof that a working tree is reusable. */ export async function reserveWarmNativeInstructionDirectory(input: { companyId: string; agentId: string; previousRunId: string; runId: string; target?: AdapterExecutionTarget | null; + observabilityCompatibilityKey?: string; canReuse: () => Promise; }): Promise<{ reuseRunId: string; adopt: (root: string, collectStopped: () => Promise) => void; release: () => Promise } | null> { for (const [id, entry] of warmNativeSessions) { @@ -580,7 +584,7 @@ export async function reserveWarmNativeInstructionDirectory(input: { }; let reusable = false; const targetProven = input.target?.kind !== "remote" || input.target.transport !== "sandbox" || Boolean(input.target.sandboxLeaseAcquisition?.providerLeaseId); - try { reusable = targetProven && !entry.closeOnReleaseReason && entry.instructionCopy.targetIdentity === instructionTargetIdentity(input.target) && await input.canReuse(); } + try { reusable = targetProven && entry.observabilityCompatibilityKey === input.observabilityCompatibilityKey && !entry.closeOnReleaseReason && entry.instructionCopy.targetIdentity === instructionTargetIdentity(input.target) && await input.canReuse(); } finally { if (!reusable) { await retire("managed agent files changed or environment replaced"); @@ -6073,6 +6077,7 @@ function warmNativeProcessConfigDigest(input: Parameters Promise; dotWorkspaceRoot?: string; + observability?: ProviderObservabilityLaunch; runnerEnvironment?: NodeJS.ProcessEnv; prepareRunnerEnvironment?: () => Promise; /** Private grant materialization; never a user-configured host path. */ @@ -12980,6 +13003,7 @@ async function createRunnerdBackendWithinSessionClaim( runnerInstanceId: retainedTransition?.runnerInstanceId ?? input.runnerInstanceId, environment: effectiveRunnerEnvironment, + observability: input.observability, workingDirectoryAuthority: remoteTarget ? "remote_runner" : "local_filesystem", @@ -13128,6 +13152,7 @@ async function createRunnerdBackendWithinSessionClaim( : undefined, runnerReconnectGraceMs: remoteTarget ? 120_000 : undefined, environment: effectiveRunnerEnvironment, + observability: input.observability, onDiagnostic: (message) => { void input.onLog?.( "stderr", diff --git a/ui/src/api/agent-observability.ts b/ui/src/api/agent-observability.ts new file mode 100644 index 00000000000..32b2ea5d79b --- /dev/null +++ b/ui/src/api/agent-observability.ts @@ -0,0 +1,10 @@ +import type { AgentObservabilitySettingsResponse, AgentObservabilityTestReceipt, UpdateAgentObservabilityRequest } from "@paperclipai/shared"; +import { api } from "./client"; + +const path = (companyId: string) => `/companies/${companyId}/observability`; +export const agentObservabilityApi = { + settings: (companyId: string) => api.get(path(companyId)), + update: (companyId: string, input: UpdateAgentObservabilityRequest) => api.put(path(companyId), input), + remove: (companyId: string, expectedRevision: string) => api.deleteWithBody(path(companyId), { expectedRevision }), + test: (companyId: string, expectedRevision: string) => api.post(`${path(companyId)}/test`, { expectedRevision }), +}; diff --git a/ui/src/components/observability/AgentObservabilitySettings.test.tsx b/ui/src/components/observability/AgentObservabilitySettings.test.tsx new file mode 100644 index 00000000000..ec78d206361 --- /dev/null +++ b/ui/src/components/observability/AgentObservabilitySettings.test.tsx @@ -0,0 +1,98 @@ +// @vitest-environment jsdom +import { createRoot, type Root } from "react-dom/client"; +import { flushSync } from "react-dom"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE, type AgentObservabilitySettings } from "@paperclipai/shared"; +import { AgentObservabilitySettingsView, type AgentObservabilitySettingsViewProps } from "./AgentObservabilitySettings"; + +let root: Root, element: HTMLDivElement; +const saved: AgentObservabilitySettings = { + companyId: "10000000-0000-4000-8000-000000000001", revision: "20000000-0000-4000-8000-000000000001", + enabled: false, destination: { provider: "braintrust", projectId: "30000000-0000-4000-8000-000000000001" }, + capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, credentialsConfigured: true, headerNames: ["authorization"], updatedAt: "2026-10-19T12:00:00Z", lastTest: null, +}; +const onSave = vi.fn(); +const props = (overrides: Partial = {}): AgentObservabilitySettingsViewProps => ({ companyId: saved.companyId, settings: saved, onSave, onTest: vi.fn(), onRemove: vi.fn(), ...overrides }); +async function render(value: AgentObservabilitySettingsViewProps) { + flushSync(() => root.render()); + await new Promise(resolve => setTimeout(resolve, 0)); + flushSync(() => {}); +} +function button(text: string) { return [...element.querySelectorAll("button")].find(node => node.textContent === text)!; } +function fill(selector: string, value: string) { + const input = element.querySelector(selector)!; + flushSync(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); +} +function click(node: HTMLElement) { flushSync(() => node.click()); } +beforeEach(() => { element = document.createElement("div"); document.body.append(element); root = createRoot(element); onSave.mockReset(); }); +afterEach(() => { flushSync(() => root.unmount()); element.remove(); }); +it("keeps saved credentials write-only and erases a cancelled replacement", async () => { + await render(props()); expect(element.querySelector('input[type="password"]')).toBeNull(); + click(button("Replace API key")); fill('input[type="password"]', "secret-input-canary"); + click(button("Cancel")); expect(element.querySelector('input[type="password"]')).toBeNull(); + expect(onSave).not.toHaveBeenCalled(); expect(element.innerHTML).not.toContain("secret-input-canary"); +}); +it("retains the saved key when enabling and includes the saved revision", async () => { + await render(props()); click(element.querySelector('[role="switch"][aria-label="Enable export"]')!); click(button("Save changes")); + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ expectedRevision: saved.revision, enabled: true, credentials: { mode: "keep" } })); +}); +it("can disable export after the saved key has been removed", async () => { + await render(props({ settings: { ...saved, enabled: true, credentialsConfigured: false } })); + click(button("Disable export")); + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ expectedRevision: saved.revision, enabled: false, credentials: { mode: "keep" } })); +}); +it("sends a complete custom header set without fetching retained values", async () => { + const custom: AgentObservabilitySettings = { ...saved, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example/otel" }, headerNames: ["authorization", "x-remove"] }; + await render(props({ settings: custom })); click(element.querySelector('button[aria-label="Remove x-remove"]')!); click(button("Save changes")); + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ credentials: { mode: "replace", headers: [{ name: "authorization" }] } })); +}); +it("preserves an unsaved secret on conflict, prevents resubmission, then clears on reload", async () => { + const reload = vi.fn(); await render(props()); click(button("Replace API key")); fill('input[type="password"]', "draft-secret-canary"); + await render(props({ conflict: true, error: "Another administrator changed these settings", onReload: reload })); + expect(element.querySelector('input[type="password"]')!.value).toBe("draft-secret-canary"); + expect(button("Save changes").disabled).toBe(true); click(button("Reload saved settings")); expect(reload).toHaveBeenCalledOnce(); + expect(element.querySelector('input[type="password"]')).toBeNull(); +}); +it("preserves a draft when a background read finds another administrator's revision", async () => { + await render(props()); click(button("Replace API key")); fill('input[type="password"]', "background-draft-canary"); + fill('#agent-observability-project', "30000000-0000-4000-8000-000000000002"); + const newer = { ...saved, revision: "20000000-0000-4000-8000-000000000002", enabled: true }; + await render(props({ settings: newer })); + expect(element.querySelector('input[type="password"]')!.value).toBe("background-draft-canary"); + expect(element.querySelector('#agent-observability-project')!.value).toBe("30000000-0000-4000-8000-000000000002"); + expect(button("Save changes").disabled).toBe(true); + expect(element.textContent).toContain("Another administrator changed these settings"); + click(button("Reload saved settings")); + expect(element.querySelector('input[type="password"]')).toBeNull(); + expect(element.querySelector('#agent-observability-project')!.value).toBe(saved.destination.provider === "braintrust" ? saved.destination.projectId : ""); + expect(element.querySelector('[role="switch"]')!.getAttribute("aria-checked")).toBe("true"); +}); +it("erases entered credentials after an explicitly accepted save", async () => { + await render(props({ resetKey: 0 })); click(button("Replace API key")); fill('input[type="password"]', "saved-draft-canary"); + click(button("Save changes")); expect(onSave).toHaveBeenCalledOnce(); + await render(props({ settings: { ...saved, revision: "20000000-0000-4000-8000-000000000003" }, resetKey: 1 })); + expect(element.querySelector('input[type="password"]')).toBeNull(); + expect(element.textContent).not.toContain("Another administrator"); +}); +it("requires re-entry of saved custom headers when the endpoint origin changes", async () => { + const custom: AgentObservabilitySettings = { ...saved, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example/otel" }, headerNames: ["authorization"] }; + await render(props({ settings: custom })); + fill('#agent-observability-endpoint', "https://another.example/otel"); click(button("Save changes")); + expect(onSave).not.toHaveBeenCalled(); expect(element.textContent).toContain("Re-enter every header value"); + fill('input[type="password"]', "new-origin-credential"); click(button("Save changes")); + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ credentials: { mode: "replace", headers: [{ name: "authorization", value: "new-origin-credential" }] } })); +}); +it("erases the previous company draft and receipt when company changes", async () => { + await render(props()); click(button("Replace API key")); fill('input[type="password"]', "old-company-secret-canary"); + await render(props({ companyId: "another-company", settings: null })); + expect(element.querySelector('input[type="password"]')!.value).toBe(""); + expect(element.querySelector('#agent-observability-project')!.value).toBe(""); + expect(element.querySelector('[role="switch"]')!.getAttribute("aria-checked")).toBe("false"); +}); +it("hides form controls without management access", async () => { + await render(props({ canManage: false, settings: null })); expect(element.querySelector("input")).toBeNull(); expect(element.textContent).toContain("Manage connections permission is required"); +}); diff --git a/ui/src/components/observability/AgentObservabilitySettings.tsx b/ui/src/components/observability/AgentObservabilitySettings.tsx new file mode 100644 index 00000000000..88ce70645e2 --- /dev/null +++ b/ui/src/components/observability/AgentObservabilitySettings.tsx @@ -0,0 +1,193 @@ +import { useEffect, useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + DEFAULT_AGENT_OBSERVABILITY_CAPTURE, updateAgentObservabilitySchema, isSameOtlpOrigin, + type AgentObservabilityCapture, type AgentObservabilitySettings, type AgentObservabilitySettingsResponse, + type AgentObservabilityTestReceipt, type UpdateAgentObservabilityRequest, +} from "@paperclipai/shared"; +import { agentObservabilityApi } from "@/api/agent-observability"; +import { ApiError } from "@/api/client"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { ToggleField } from "@/components/agent-config-primitives"; + +type HeaderDraft = { id: number; originalName: string | null; name: string; value: string }; +const captureLabels: Array<{ key: keyof AgentObservabilityCapture; label: string }> = [ + { key: "userPrompts", label: "User prompts" }, + { key: "assistantResponses", label: "Assistant responses" }, + { key: "toolContent", label: "Tool output and content" }, + { key: "toolDetails", label: "Additional tool details (Claude Code)" }, +]; +export interface AgentObservabilitySettingsViewProps { + companyId: string; + settings: AgentObservabilitySettings | null; + canManage?: boolean; + saving?: boolean; + testing?: boolean; + removing?: boolean; + error?: string | null; + conflict?: boolean; + result?: AgentObservabilityTestReceipt | null; + /** Changes only after an explicit successful mutation, never a background refetch. */ + resetKey?: number; + onSave: (input: UpdateAgentObservabilityRequest) => void; + onTest: () => void; + onRemove: () => void; + onReload?: () => void; +} + +export function AgentObservabilitySettingsView(props: AgentObservabilitySettingsViewProps) { + const { companyId, settings, canManage = true, saving, testing, removing, error, conflict, onSave, onTest, onRemove, onReload } = props; + const [provider, setProvider] = useState<"braintrust" | "otlp">("braintrust"); + const [projectId, setProjectId] = useState(""); + const [endpoint, setEndpoint] = useState(""); + const [enabled, setEnabled] = useState(false); + const [capture, setCapture] = useState({ ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }); + const [apiKey, setApiKey] = useState(""); + const [replaceKey, setReplaceKey] = useState(false); + const [headers, setHeaders] = useState([]); + const [validation, setValidation] = useState(null); + const [applied, setApplied] = useState(settings); + const nextHeader = useRef(0); + const revision = settings?.revision ?? null; + function reset() { + setApplied(settings); + setProvider(settings?.destination.provider ?? "braintrust"); + setProjectId(settings?.destination.provider === "braintrust" ? settings.destination.projectId : ""); + setEndpoint(settings?.destination.provider === "otlp" ? settings.destination.endpointBaseUrl : ""); + setEnabled(settings?.enabled ?? false); + setCapture({ ...(settings?.capture ?? DEFAULT_AGENT_OBSERVABILITY_CAPTURE) }); + setApiKey(""); setReplaceKey(false); setValidation(null); + setHeaders((settings?.destination.provider === "otlp" ? settings.headerNames : []).map(name => ({ id: nextHeader.current++, originalName: name, name, value: "" }))); + } + const original = applied; + const headersChanged = provider === "otlp" && (headers.some(row => row.value || row.originalName !== row.name.toLowerCase()) + || JSON.stringify(headers.map(row => row.name.toLowerCase()).sort()) !== JSON.stringify(original?.destination.provider === "otlp" ? [...original.headerNames].sort() : [])); + const dirty = !original || provider !== original.destination.provider || enabled !== original.enabled || replaceKey || Boolean(apiKey) || headersChanged + || JSON.stringify(capture) !== JSON.stringify(original.capture) + || (provider === "braintrust" ? projectId !== (original.destination.provider === "braintrust" ? original.destination.projectId : "") + : endpoint !== (original.destination.provider === "otlp" ? original.destination.endpointBaseUrl : "")); + // Explicit mutation success/company changes erase secrets. Background reads + // may update a clean form, but cannot overwrite an unsaved draft. + useEffect(() => { reset(); }, [companyId, props.resetKey]); + useEffect(() => { if (!dirty && !conflict) reset(); }, [revision]); + const hasConflict = Boolean(conflict || revision !== (original?.revision ?? null)); + const requireFreshHeaders = provider === "otlp" && original?.destination.provider === "otlp" + && original.headerNames.length > 0 && !isSameOtlpOrigin(original.destination, { provider: "otlp", endpointBaseUrl: endpoint }); + const busy = saving || testing || removing; + const providerChanged = !original || original.destination.provider !== provider; + function save(overrideEnabled?: boolean) { + if (requireFreshHeaders && headers.some(row => !row.value)) { + setValidation("Re-enter every header value for the new endpoint, or remove the header."); return; + } + const common = { expectedRevision: original?.revision ?? null, enabled: overrideEnabled ?? enabled, capture }; + const input = provider === "braintrust" + ? { ...common, destination: { provider, projectId }, credentials: providerChanged || replaceKey || apiKey || (!original?.credentialsConfigured && common.enabled) ? { mode: "replace", apiKey } : { mode: "keep" } } + : { ...common, destination: { provider, endpointBaseUrl: endpoint }, credentials: providerChanged || headersChanged || requireFreshHeaders + ? { mode: "replace", headers: headers.map(row => ({ name: row.name, ...(row.value ? { value: row.value } : {}) })) } : { mode: "keep" } }; + const parsed = updateAgentObservabilitySchema.safeParse(input); + if (!parsed.success) { setValidation(parsed.error.issues[0]?.message ?? "Review the configuration."); return; } + setValidation(null); onSave(parsed.data); + } + const receipt = props.result?.revision === revision ? props.result : settings?.lastTest; + return
+
+

Agent observability

+

Send Codex and Claude Code logs and traces to your observability service.

+
+ {!canManage ?

Manage connections permission is required to configure agent observability.

: <> +
+
+ + +
+ {provider === "braintrust" ? <> +
+ setProjectId(event.target.value)} placeholder="Braintrust project UUID" autoComplete="off" /> +
+
+ + {!providerChanged && original?.credentialsConfigured && !replaceKey ?
+ API key configured +
: setApiKey(event.target.value)} autoComplete="new-password" />} +
+ : <> +
+ setEndpoint(event.target.value)} placeholder="https://collector.example.com" autoComplete="off" /> +

Use the base URL without /v1/logs or /v1/traces. It must be reachable from each agent environment.

+
+
+
+ {headers.length === 0 &&

No authentication headers.

} + {headers.map(row =>
+ setHeaders(rows => rows.map(item => item.id === row.id ? { ...item, name: event.target.value } : item))} autoComplete="off" /> + setHeaders(rows => rows.map(item => item.id === row.id ? { ...item, value: event.target.value } : item))} autoComplete="new-password" /> + +
)} +
+ } +

The agent harness receives these credentials. Use a key limited to telemetry ingestion.

+ +
Content capture +
{captureLabels.map(row => setCapture(current => ({ ...current, [row.key]: value }))} />)} +

Native limits apply. Exported records can contain task content, including prompts and tool output. A collector can forward records to multiple services.

+
+
+
+ {(validation || error || hasConflict) &&

{validation || error || "Another administrator changed these settings. Reload the saved settings to continue."}

} + {hasConflict && } + {!dirty && receipt &&
+

{receipt.status === "accepted" ? "Test payload accepted" : receipt.status === "partial" ? "Some test records were rejected" : "Test payload failed"}

+

{new Date(receipt.finishedAt).toLocaleString()} · Sent from the Paperclip server.

+ {(["logs", "traces"] as const).map(signal => receipt.signals[signal].message &&

{signal === "logs" ? "Logs" : "Traces"}: {receipt.signals[signal].message}

)} +
} +
+ + +
+ {dirty && settings &&

Save changes before testing.

} + {settings &&
+ + {settings.enabled && } + +
} + } +
; +} + +export function AgentObservabilitySettingsSection({ companyId }: { companyId: string }) { + return ; +} +function AgentObservabilitySettingsContainer({ companyId }: { companyId: string }) { + const client = useQueryClient(); + const queryKey = ["agent-observability", companyId]; + const query = useQuery({ queryKey, queryFn: () => agentObservabilityApi.settings(companyId) }); + const [error, setError] = useState(null); + const [conflict, setConflict] = useState(false); + const [resetKey, setResetKey] = useState(0); + const accept = (data: AgentObservabilitySettingsResponse) => { client.setQueryData(queryKey, data); setError(null); setConflict(false); setResetKey(value => value + 1); }; + const reject = (cause: unknown) => { + const stale = cause instanceof ApiError && cause.status === 409; + setConflict(stale); + setError(stale ? "Another administrator changed these settings. Reload the saved settings to continue." : cause instanceof Error ? cause.message : "The observability settings could not be updated."); + }; + const save = useMutation({ gcTime: 0, mutationFn: (input: UpdateAgentObservabilityRequest) => agentObservabilityApi.update(companyId, input), onSuccess: accept, onError: reject, onSettled: () => queueMicrotask(() => save.reset()) }); + const remove = useMutation({ mutationFn: (revision: string) => agentObservabilityApi.remove(companyId, revision), onSuccess: accept, onError: reject }); + const test = useMutation({ mutationFn: (revision: string) => agentObservabilityApi.test(companyId, revision), onSuccess: receipt => { + setError(null); client.setQueryData(queryKey, current => current?.settings?.revision === receipt.revision + ? { ...current, settings: { ...current.settings, lastTest: receipt } } : current); + }, onError: reject }); + useEffect(() => { setError(null); setConflict(false); save.reset(); test.reset(); remove.reset(); }, [companyId]); + if (query.isPending) return

Loading agent observability…

; + if (!query.data) return

Agent observability settings could not be loaded.

; + return save.mutate(input)} onTest={() => { if (query.data.settings) test.mutate(query.data.settings.revision); }} + onRemove={() => { if (query.data.settings) remove.mutate(query.data.settings.revision); }} + onReload={() => { setConflict(false); setError(null); void query.refetch(); }} />; +} diff --git a/ui/src/pages/CompanySettings.tsx b/ui/src/pages/CompanySettings.tsx index a83f33c2f39..5abf250f442 100644 --- a/ui/src/pages/CompanySettings.tsx +++ b/ui/src/pages/CompanySettings.tsx @@ -1,4 +1,5 @@ import { ChangeEvent, useEffect, useState } from "react"; +import { AgentObservabilitySettingsSection } from "../components/observability/AgentObservabilitySettings"; import { useNavigate } from "react-router-dom"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { @@ -375,6 +376,8 @@ export function CompanySettings() { )} + + {/* Danger Zone */} diff --git a/ui/storybook/stories/companies/observability/01-settings.stories.tsx b/ui/storybook/stories/companies/observability/01-settings.stories.tsx new file mode 100644 index 00000000000..d03f3d84204 --- /dev/null +++ b/ui/storybook/stories/companies/observability/01-settings.stories.tsx @@ -0,0 +1,32 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, fn, userEvent, within } from "storybook/test"; +import { AgentObservabilitySettingsView } from "@/components/observability/AgentObservabilitySettings"; +import { companyId, configured, custom } from "../../../support/observability/fixtures"; + +const meta = { + id: "agent-observability-01-settings", + title: "Getting started and access/Companies and portability/Agent observability/01 Settings", + component: AgentObservabilitySettingsView, + decorators: [Story =>
], + args: { companyId, settings: configured, onSave: fn(), onTest: fn(), onRemove: fn() }, + parameters: { paperclip: { capabilities: ["companies/agent-observability"], implementation: "production", boundary: "fixtures", + notes: "Production settings view: initial setup, destinations, stored credentials, content controls, and narrow layout. Provider and authorization behavior are simulated." } }, +} satisfies Meta; +export default meta; +type Story = StoryObj; +export const Empty: Story = { args: { settings: null } }; +export const Braintrust: Story = {}; +export const CustomAuthenticated: Story = { args: { settings: custom } }; +export const CustomAnonymous: Story = { args: { settings: { ...custom, credentialsConfigured: false, headerNames: [] } } }; +export const Disabled: Story = { args: { settings: { ...configured, enabled: false } } }; +export const ReplaceCredential: Story = { play: async ({ canvasElement }) => { + const canvas = within(canvasElement); await userEvent.click(canvas.getByRole("button", { name: "Replace API key" })); + await userEvent.type(canvas.getByLabelText("API key"), "fake-storybook-key"); + await expect(canvas.getByRole("button", { name: "Save changes" })).toBeEnabled(); +} }; +export const EditHeaders: Story = { args: { settings: custom }, play: async ({ canvasElement }) => { + const canvas = within(canvasElement); await userEvent.click(canvas.getByRole("button", { name: "Remove x-team" })); + await userEvent.click(canvas.getByRole("button", { name: "Add header" })); + await userEvent.type(canvas.getAllByLabelText("Header name")[1]!, "x-project"); +} }; +export const Mobile: Story = { globals: { viewport: { value: "mobile1", isRotated: false } } }; diff --git a/ui/storybook/stories/companies/observability/02-states.stories.tsx b/ui/storybook/stories/companies/observability/02-states.stories.tsx new file mode 100644 index 00000000000..068a561cf57 --- /dev/null +++ b/ui/storybook/stories/companies/observability/02-states.stories.tsx @@ -0,0 +1,22 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { fn } from "storybook/test"; +import { AgentObservabilitySettingsView } from "@/components/observability/AgentObservabilitySettings"; +import { companyId, configured, receipt, failedReceipt } from "../../../support/observability/fixtures"; +const meta = { + id: "agent-observability-02-states", + title: "Getting started and access/Companies and portability/Agent observability/02 States", + component: AgentObservabilitySettingsView, + decorators: [Story =>
], + args: { companyId, settings: configured, onSave: fn(), onTest: fn(), onRemove: fn(), onReload: fn() }, + parameters: { paperclip: { capabilities: ["companies/agent-observability"], implementation: "production", boundary: "fixtures", + notes: "Production view with saving, testing, accepted/partial/failed receipts, stale revisions, and restricted access. Receipts do not establish real destination delivery." } }, +} satisfies Meta; +export default meta; +type Story = StoryObj; +export const Saving: Story = { args: { saving: true } }; +export const Testing: Story = { args: { testing: true } }; +export const Accepted: Story = { args: { result: receipt } }; +export const Rejected: Story = { args: { result: failedReceipt } }; +export const Partial: Story = { args: { result: { ...receipt, status: "partial", signals: { logs: receipt.signals.logs, traces: failedReceipt.signals.traces } } } }; +export const Conflict: Story = { args: { conflict: true, error: "Another administrator changed these settings. Reload the saved settings to continue." } }; +export const Restricted: Story = { args: { settings: null, canManage: false } }; diff --git a/ui/storybook/stories/companies/observability/03-journey.stories.tsx b/ui/storybook/stories/companies/observability/03-journey.stories.tsx new file mode 100644 index 00000000000..d05f009071c --- /dev/null +++ b/ui/storybook/stories/companies/observability/03-journey.stories.tsx @@ -0,0 +1,72 @@ +import { useLayoutEffect, useState } from "react"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, userEvent, waitFor, within } from "storybook/test"; +import { useQueryClient } from "@tanstack/react-query"; +import type { AgentObservabilitySettings, UpdateAgentObservabilityRequest } from "@paperclipai/shared"; +import { Route, Routes } from "@/lib/router"; +import { Layout } from "@/components/Layout"; +import { CompanySettings } from "@/pages/CompanySettings"; +import { PluginLauncherProvider } from "@/plugins/launchers"; +import { unconfigured as decisionSettings } from "../../../support/decision-models/fixtures"; +import { companyId, configured, receipt } from "../../../support/observability/fixtures"; + +function Journey() { + const client = useQueryClient(); + const [ready, setReady] = useState(false); + useLayoutEffect(() => { + const original = window.fetch; + let settings: AgentObservabilitySettings | null = null; + let version = 1; + window.fetch = async (input, init) => { + const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, window.location.origin); + const base = `/api/companies/${companyId}/observability`; + if (url.pathname === `/api/companies/${companyId}/decision-model`) return Response.json({ canManage: true, settings: decisionSettings, choices: [] }); + if (url.pathname === "/api/instance/settings/general") return Response.json({ censorUsernameInLogs: false, feedbackDataSharingPreference: "prompt" }); + if (url.pathname === base) { + if (init?.method === "PUT") { + const data = JSON.parse(String(init.body)) as UpdateAgentObservabilityRequest; + if (data.expectedRevision !== (settings?.revision ?? null)) return Response.json({ error: "Settings changed" }, { status: 409 }); + settings = { ...configured, revision: `10000000-0000-4000-8000-${String(version++).padStart(12, "0")}`, + enabled: data.enabled, destination: data.destination, capture: data.capture, lastTest: null }; + } else if (init?.method === "DELETE") settings = null; + return Response.json({ canManage: true, settings }); + } + if (url.pathname === `${base}/test` && settings) { + settings = { ...settings, lastTest: { ...receipt, revision: settings.revision } }; + return Response.json(settings.lastTest); + } + return original(input, init); + }; + setReady(true); + return () => { window.fetch = original; client.removeQueries({ queryKey: ["agent-observability", companyId] }); }; + }, [client]); + return ready ? }> + } /> + : null; +} +const meta = { + id: "agent-observability-03-journey", + title: "Getting started and access/Companies and portability/Agent observability/03 Journey", + component: Journey, + parameters: { layout: "fullscreen", initialEntries: ["/PAP/company/settings"], paperclip: { + capabilities: ["companies/agent-observability"], implementation: "production", boundary: "fixtures", + notes: "Production company settings and app shell. Stateful mocked API verifies configure, save, test, disable, and remove; no real OTLP or credentials." } }, +} satisfies Meta; +export default meta; +type Story = StoryObj; +export const ConfigureTestDisable: Story = { play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await waitFor(() => expect(canvas.getByRole("heading", { name: "Agent observability" })).toBeVisible()); + const section = within(canvas.getByRole("heading", { name: "Agent observability" }).closest("section")!); + await userEvent.type(section.getByLabelText("Project ID"), "20000000-0000-4000-8000-000000000001"); + await userEvent.type(section.getByLabelText("API key"), "fake-storybook-key"); + await userEvent.click(section.getByRole("switch", { name: "Enable export" })); + await userEvent.click(section.getByRole("button", { name: "Save changes" })); + await waitFor(() => expect(section.getByText("API key configured")).toBeVisible()); + await userEvent.click(section.getByRole("button", { name: "Test saved settings" })); + await waitFor(() => expect(section.getByText("Test payload accepted")).toBeVisible()); + await userEvent.click(section.getByRole("button", { name: "Disable export" })); + await waitFor(() => expect(section.queryByRole("button", { name: "Disable export" })).not.toBeInTheDocument()); + await userEvent.click(section.getByRole("button", { name: "Remove configuration" })); + await waitFor(() => expect(section.queryByText("API key configured")).not.toBeInTheDocument()); +} }; diff --git a/ui/storybook/support/observability/fixtures.ts b/ui/storybook/support/observability/fixtures.ts new file mode 100644 index 00000000000..ccaa4ed2725 --- /dev/null +++ b/ui/storybook/support/observability/fixtures.ts @@ -0,0 +1,19 @@ +import { DEFAULT_AGENT_OBSERVABILITY_CAPTURE, type AgentObservabilitySettings, type AgentObservabilityTestReceipt } from "@paperclipai/shared"; +export const companyId = "company-storybook"; +export const configured: AgentObservabilitySettings = { + companyId, revision: "10000000-0000-4000-8000-000000000001", enabled: true, + destination: { provider: "braintrust", projectId: "20000000-0000-4000-8000-000000000001" }, + capture: { ...DEFAULT_AGENT_OBSERVABILITY_CAPTURE }, credentialsConfigured: true, headerNames: ["authorization"], + updatedAt: "2026-10-19T12:00:00Z", lastTest: null, +}; +export const custom: AgentObservabilitySettings = { ...configured, destination: { provider: "otlp", endpointBaseUrl: "https://collector.example.com/otel" }, headerNames: ["authorization", "x-team"] }; +export const receipt: AgentObservabilityTestReceipt = { + revision: configured.revision, marker: "paperclip-observability-test-storybook", source: "control_plane", + startedAt: "2026-10-19T12:01:00Z", finishedAt: "2026-10-19T12:01:01Z", status: "accepted", + signals: { logs: { status: "accepted", httpStatus: 200, rejectedRecords: 0, errorCode: null, message: null, durationMs: 125 }, + traces: { status: "accepted", httpStatus: 200, rejectedRecords: 0, errorCode: null, message: null, durationMs: 140 } }, +}; +export const failedReceipt: AgentObservabilityTestReceipt = { ...receipt, status: "failed", signals: { + logs: { ...receipt.signals.logs, status: "failed", httpStatus: 403, errorCode: "receiver_rejected", message: "The receiver rejected the test records. Check credentials and destination routing." }, + traces: { ...receipt.signals.traces, status: "failed", httpStatus: 403, errorCode: "receiver_rejected", message: "The receiver rejected the test records. Check credentials and destination routing." }, +} };