From d4892d476138e3103f1a8a7fbdeecd0ab4073bb9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:49:08 -0500 Subject: [PATCH] fix(observability): sample native runs independently of triggers (#16285) Start native background run traces under the operator's root sampling policy and link them to valid trigger contexts. Preserve child parenting and silent unconfigured tracing. Co-Authored-By: Paperclip --- doc/observability.md | 6 ++ .../native-run-trace-context.test.ts | 75 +++++++++++++++++++ .../native-runtime/native-run-trace.ts | 5 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 server/src/services/native-runtime/native-run-trace-context.test.ts diff --git a/doc/observability.md b/doc/observability.md index 03495423735..bdfce422305 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -204,6 +204,12 @@ ends before outer workspace persistence. The heartbeat-owned local run-log every other native span carries a real OpenTelemetry parent context rather than only a descriptive `parentName` field. +The native lifecycle root uses the operator's root sampling policy. It links +to a valid active trigger trace without inheriting its sampling decision. +An unsampled HTTP request therefore cannot suppress an otherwise sampled +background run. An operator who disables root sampling still gets no spans. +Without an OTLP endpoint, tracing remains silent and does not export data. + The run link in Honeycomb matches both `task.provider_session` for native runs and `task.run` for legacy adapters. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the two root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged. The canonical lifecycle is: diff --git a/server/src/services/native-runtime/native-run-trace-context.test.ts b/server/src/services/native-runtime/native-run-trace-context.test.ts new file mode 100644 index 00000000000..6277ba82e61 --- /dev/null +++ b/server/src/services/native-runtime/native-run-trace-context.test.ts @@ -0,0 +1,75 @@ +import { context, trace, TraceFlags } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { AlwaysOffSampler, ParentBasedSampler, TracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace"; +import { expect, it, vi } from "vitest"; +import { createNativeRunTrace } from "./native-run-trace.js"; + +it.each([TraceFlags.NONE, TraceFlags.SAMPLED])("exports a complete native run independently of trigger flags %i", async traceFlags => { + const manager = new AsyncLocalStorageContextManager().enable(); + const exporter = new InMemorySpanExporter(); + const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] }); + expect(context.setGlobalContextManager(manager)).toBe(true); + expect(trace.setGlobalTracerProvider(provider)).toBe(true); + const trigger = { + traceId: "11111111111111111111111111111111", spanId: "2222222222222222", traceFlags, isRemote: true, + }; + try { + await context.with(trace.setSpanContext(context.active(), trigger), async () => { + const native = createNativeRunTrace({ runId: "diagnostic-native-run" }); + await native.measure("native.session.execute", async () => undefined); + await native.finish("ok"); + expect(trace.getActiveSpan()?.spanContext()).toEqual(trigger); + }); + await provider.forceFlush(); + const spans = exporter.getFinishedSpans(); + expect(spans.map(span => span.name).sort()).toEqual(["native.session.execute", "task.provider_session"]); + const root = spans.find(span => span.name === "task.provider_session")!; + const child = spans.find(span => span.name === "native.session.execute")!; + expect(root.parentSpanContext).toBeUndefined(); + expect(root.links.map(link => link.context)).toEqual([trigger]); + expect(root.spanContext().traceId).not.toBe(trigger.traceId); + expect(child.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + expect(child.spanContext().traceId).toBe(root.spanContext().traceId); + } finally { + trace.disable(); context.disable(); manager.disable(); await provider.shutdown(); + } +}); + +it("honors an operator who disables root sampling despite a sampled trigger", async () => { + const manager = new AsyncLocalStorageContextManager().enable(); + const exporter = new InMemorySpanExporter(); + const provider = new TracerProvider({ + sampler: new ParentBasedSampler({ root: new AlwaysOffSampler() }), + spanProcessors: [new SimpleSpanProcessor({ exporter })], + }); + expect(context.setGlobalContextManager(manager)).toBe(true); + expect(trace.setGlobalTracerProvider(provider)).toBe(true); + try { + await context.with(trace.setSpanContext(context.active(), { + traceId: "11111111111111111111111111111111", spanId: "2222222222222222", traceFlags: TraceFlags.SAMPLED, isRemote: true, + }), async () => { + const native = createNativeRunTrace({ runId: "disabled-native-run" }); + await native.measure("native.session.execute", async () => undefined); + await native.finish("ok"); + }); + await provider.forceFlush(); + expect(exporter.getFinishedSpans()).toEqual([]); + } finally { + trace.disable(); context.disable(); manager.disable(); await provider.shutdown(); + } +}); + +it("stays silent without a tracer provider", async () => { + trace.disable(); context.disable(); + const methods = ["log", "info", "warn", "error", "debug"] as const; + const spies = methods.map(method => vi.spyOn(console, method)); + try { + const native = createNativeRunTrace({ runId: "unconfigured-native-run" }); + const value = {}; + await expect(native.measure("native.session.execute", async () => value)).resolves.toBe(value); + const error = new Error("original failure"); + await expect(native.measure("native.session.execute", async () => { throw error; })).rejects.toBe(error); + await native.finish("ok"); + for (const spy of spies) expect(spy).not.toHaveBeenCalled(); + } finally { for (const spy of spies) spy.mockRestore(); } +}); diff --git a/server/src/services/native-runtime/native-run-trace.ts b/server/src/services/native-runtime/native-run-trace.ts index e3f4f8291b0..83080774d83 100644 --- a/server/src/services/native-runtime/native-run-trace.ts +++ b/server/src/services/native-runtime/native-run-trace.ts @@ -1,5 +1,6 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { createHash } from "node:crypto"; +import { isSpanContextValid, ROOT_CONTEXT, trace } from "@opentelemetry/api"; import { runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import type { AdapterRuntimeEvent } from "../../adapters/index.js"; @@ -210,6 +211,7 @@ export function createNativeRunTrace(input: { const runIdHash = hashedId(input.runId); let rootSpan: SpanHandle; try { + const trigger = trace.getActiveSpan()?.spanContext(); rootSpan = tracing.tracer.startSpan("task.provider_session", { startTime: traceStartedAtMs, attributes: { @@ -218,7 +220,8 @@ export function createNativeRunTrace(input: { "paperclip.task.run.trace_schema_version": NATIVE_RUN_TRACE_SCHEMA_VERSION, }, - }); + ...(trigger && isSpanContextValid(trigger) ? { links: [{ context: trigger }] } : {}), + }, ROOT_CONTEXT); } catch { rootSpan = NOOP_SPAN; }