diff --git a/doc/observability.md b/doc/observability.md index 8a379342a1d..93513a80716 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -415,6 +415,37 @@ values become `unknown`. A signal such as `SIGKILL` does not establish who sent it or prove an out-of-memory kill. These fields do not change error grouping or run outcomes, and do not include process output or adapter result payloads. +The shared reporter also attaches bounded diagnostic contexts for both legacy +and native runs: + +- `run_execution`: runtime mode, execution stage/native phase, driver and version, + duration, failure phase, stop reason, error family, and timeout settings when available. +- `adapter_failure`: selected adapter error fields such as phase, category, + protocol code, retryability, cause message, stack preview, HTTP status, and request ID. +- `provider_failure`: the saved provider failure category, title, and details. +- `run_exception_0` through `run_exception_3`: exception names, codes, HTTP + statuses, and request IDs for a caught exception and up to three causes. + +The execution and setup catch paths pass the original exception to the reporter. +It snapshots and rebuilds only these selected fields and the original message and +stack. Sentry receives the sanitized cause chain rather than a stack created at +the reporting call. Saved adapter results use an available adapter stack preview; +without one, the report has no synthetic reporting stack. + +Credential patterns, the current user's home path, registered run-secret values, +and known host/runtime environment credentials are removed before truncation. +Declared environment secret bindings are included even when their key has no +credential-like name. Runtime values are used only for redaction and are never +copied into an event. Stack fields are limited to 8,192 characters, +exception messages to 2,048, provider titles to 4,096, and provider details to +12,288. Other diagnostic strings are limited to 200 characters (adapter cause +messages: 2,048). Truncation is marked in the text and in +`run_execution.truncatedFields`; cyclic and deeper cause chains are marked too. +Request/response objects, headers, environment/configuration, prompts, stdout, +stderr, and arbitrary adapter result fields are not copied. The existing DSN +opt-in gate and error-code/adapter fingerprint remain unchanged. These reports +cannot recover diagnostic data that a provider or adapter discarded upstream. + ### Browser data The browser sends no page URL, no referrer, no user agent, and no diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 39d9e150fe8..6798347819f 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1048,7 +1048,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { try { await Promise.all([...fixtureServices].map((service) => service.shutdown())); } finally { - await retireFixtureState([...fixtureCompanies]); + const companyIds = [...fixtureCompanies]; + await retireFixtureState(companyIds); + if (companyIds.length > 0) { + // Pausing an endpoint does not remove its rows from global recovery + // selectors. After every assertion and worker shutdown, settle leftover + // fixture work so later cases cannot claim its leases or retry its I/O. + await db.update(chatActions).set({ status: "cancelled" }) + .where(and(inArray(chatActions.companyId, companyIds), notInArray(chatActions.status, ["processed", "cancelled"]))); + await db.update(chatDeliveries).set({ state: "failed", nextAttemptAt: null }) + .where(and(inArray(chatDeliveries.companyId, companyIds), inArray(chatDeliveries.state, ["received", "processing", "retry"]))); + await db.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null }) + .where(and(inArray(chatPublications.companyId, companyIds), inArray(chatPublications.state, ["pending", "awaiting_consent", "streaming", "retry"]))); + } fixtureServices.clear(); fixtureCompanies.clear(); } diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 3d95884685a..f6d391e0d56 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1565,8 +1565,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { "continuation_task_ownership_changed", ])("retains untyped continuation setup failures: %s", async (message) => { const { runId, wakeupRequestId } = await seedQueuedIssueRunFixture(); + const error = new Error(message, { cause: Object.assign(new Error("upstream setup failed"), { code: "ECONNRESET" }) }); const build = vi.spyOn(executionContinuation, "buildExecutionContinuation") - .mockRejectedValueOnce(new Error(message)); + .mockRejectedValueOnce(error); try { const heartbeat = heartbeatService(db); await heartbeat.resumeQueuedRuns(); @@ -1576,6 +1577,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", errorCode: "setup_failed", error: message, }); + const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]); + expect(report?.diagnostics).toMatchObject({ + execution: { failurePhase: "setup" }, + exceptions: [{ message, stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }, { code: "ECONNRESET" }], + }); const [wakeup] = await db.select().from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, wakeupRequestId)); expect(wakeup.status).toBe("failed"); @@ -4801,6 +4807,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(failedRun?.error).toContain( "is not installed or its plugin worker is not running", ); + const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]); + expect(report?.diagnostics).toMatchObject({ + execution: { failurePhase: "execute" }, + exceptions: [{ stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }], + }); const interaction = await waitForValue(async () => { const row = await db @@ -5282,6 +5293,38 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(validationComment).toBeTruthy(); }); + it.each(["throw", "result"])("redacts opaque environment-bound credentials from Sentry diagnostics: %s", async (mode) => { + const { companyId, agentId, runId } = await seedQueuedIssueRunFixture(); + const svc = secretService(db); + const value = "opaque-runtime-credential-fixture"; + const secret = await svc.create(companyId, { + name: `sentry-redaction-${randomUUID()}`, provider: "local_encrypted", value, + }); + const env = { CUSTOM_BINDING: { type: "secret_ref", secretId: secret.id, version: "latest" } }; + await svc.syncEnvBindingsForTarget(companyId, { targetType: "agent", targetId: agentId }, env); + await db.update(agents).set({ adapterConfig: { env } }).where(eq(agents.id, agentId)); + mockAdapterExecute.mockImplementationOnce(async (input) => { + expect(input.config.env.CUSTOM_BINDING).toBe(value); + const message = `upstream rejected ${value}`; + if (mode === "throw") throw new Error(message, { cause: new Error(message) }); + return { + exitCode: 1, signal: null, timedOut: false, errorMessage: message, errorCode: "adapter_failed", + errorMeta: { causeMessage: message }, + resultJson: { terminalSessionFailure: { category: "service", details: message } }, + }; + }); + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await heartbeat.waitForRunExecutionDrain(runId); + const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]); + expect(report).toBeDefined(); + expect(report.errorMessage).toContain("upstream rejected"); + expect(JSON.stringify(report)).not.toContain(value); + if (mode === "throw") expect(report.diagnostics.exceptions).toHaveLength(2); + else expect(report.diagnostics.provider.category).toBe("service"); + }); + it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts index 5cfe758e2bb..11b3c259f58 100644 --- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts +++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts @@ -1,5 +1,7 @@ import { createRequire } from "node:module"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { collectRunFailureDiagnostics, sanitizeRunFailureDiagnostics } from "../services/run-failure-diagnostics.js"; +import type { heartbeatRuns } from "@paperclipai/db"; // Sentry is an optional peer. When installed, exercise the real SDK with an // in-memory transport, including its context behavior without an OTel manager. @@ -76,7 +78,17 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", exitCode: null, signal: "SIGKILL", }; - captureRunFailure(first); + const original = Object.assign(new Error(first.errorMessage, { cause: Object.assign(new Error("upstream connection reset"), { + code: "ECONNRESET", requestId: "request-123", status: 503, + stack: "Error: upstream connection reset\n at socketRead (/app/provider.js:19:7)", + response: { body: "private-response" }, + }) }), { stack: "Error: first run failed\n at originalAdapter (/app/adapter.js:42:7)", request: { headers: "private-headers" } }); + const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ + runtimeMode: "legacy", resultJson: { terminalSessionFailure: { + category: "service", title: "Provider failed", details: "d".repeat(9000), + } }, + } as unknown as typeof heartbeatRuns.$inferSelect, { error: original, phase: "execute" })); + captureRunFailure({ ...first, diagnostics }); await Promise.resolve(); captureException(new Error("unrelated database error")); captureRunFailure(second); @@ -91,7 +103,7 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", expect(events).toHaveLength(5); const captured = (message: string) => events.find((event) => - (event.exception as { values: Array<{ value: string }> }).values[0]?.value === message, + (event.exception as { values: Array<{ value: string }> }).values.some((value) => value.value === message), ); for (const run of [first, second]) { expect(captured(run.errorMessage)).toMatchObject({ @@ -108,10 +120,25 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", fingerprint: [first.errorCode, first.agentAdapter], }); expect(JSON.stringify(events)).not.toContain("private-signal-payload"); + const firstCapture = captured(first.errorMessage)!; + const exceptions = (firstCapture.exception as { values: Array> }).values; + expect(exceptions).toHaveLength(2); + expect(JSON.stringify(exceptions)).toContain("originalAdapter"); + expect(JSON.stringify(exceptions)).toContain("socketRead"); + expect(JSON.stringify(exceptions)).not.toContain("captureRunFailure"); + expect(JSON.stringify(captured(second.errorMessage)?.exception)).not.toContain("captureRunFailure"); + expect(firstCapture).toMatchObject({ contexts: { + provider_failure: { category: "service", details: "d".repeat(9000) }, + run_exception_1: { code: "ECONNRESET", requestId: "request-123", status: 503 }, + } }); + expect(JSON.stringify(events)).not.toContain("private-"); for (const message of ["unrelated database error", "unrelated filesystem error"]) { const event = captured(message); expect(event).toBeDefined(); expect(event).not.toHaveProperty("contexts.run_failure"); + for (const context of ["run_execution", "adapter_failure", "provider_failure", "run_exception_0", "run_exception_1"]) { + expect(event).not.toHaveProperty(`contexts.${context}`); + } expect(event).not.toHaveProperty("fingerprint"); for (const tag of ["run_id", "task_id", "error_code", "agent_adapter", "run_status"]) { expect(event).not.toHaveProperty(`tags.${tag}`); diff --git a/server/src/__tests__/run-failure-sentry.test.ts b/server/src/__tests__/run-failure-sentry.test.ts index d5285698a43..9ba4c7467f4 100644 --- a/server/src/__tests__/run-failure-sentry.test.ts +++ b/server/src/__tests__/run-failure-sentry.test.ts @@ -215,6 +215,31 @@ describe("captureRunFailure", () => { expect((received as Error).message).toBe("boom"); }); + it("uses sanitized original stacks and causes instead of the reporting stack", async () => { + const { sentryModule, captureException } = await importFreshSentryWithGateOpen(); + sentryModule.captureRunFailure(baseEvent({ diagnostics: { + execution: { runtimeMode: "native", failurePhase: "setup" }, + adapter: {}, provider: {}, truncatedFields: [], + exceptions: [ + { name: "TypeError", message: "setup failed", stack: "TypeError: setup failed\n at originalSetup (/app/setup.js:42:7)" }, + { name: "Error", message: "connection reset", stack: "Error: connection reset\n at socketRead (/app/network.js:9:4)", code: "ECONNRESET", requestId: "req-123" }, + ], + } })); + const [error, context] = captureException.mock.calls[0]!; + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ name: "TypeError", message: "setup failed", cause: { message: "connection reset" } }); + expect((error as Error).stack).toContain("originalSetup"); + expect((error as Error).stack).not.toContain("captureRunFailure"); + expect(context?.contexts.run_exception_1).toMatchObject({ code: "ECONNRESET", requestId: "req-123" }); + expect(context?.contexts.run_execution).toMatchObject({ failurePhase: "setup" }); + }); + + it("does not invent a reporter stack for a saved result with no original exception", async () => { + const { sentryModule, captureException } = await importFreshSentryWithGateOpen(); + sentryModule.captureRunFailure(baseEvent()); + expect((captureException.mock.calls[0]![0] as Error).stack).toBeUndefined(); + }); + it("does not throw and captures nothing when the gate is closed", async () => { vi.resetModules(); const sentryModule = await import("../sentry.js"); diff --git a/server/src/sentry.ts b/server/src/sentry.ts index 1a9c30f4642..ee7f28f49d5 100644 --- a/server/src/sentry.ts +++ b/server/src/sentry.ts @@ -56,6 +56,7 @@ // `instrumentation.ts`. import os from "node:os"; +import type { RunFailureDiagnostics } from "./services/run-failure-diagnostics.js"; import { readBuildCommit } from "./build-commit.js"; import { checkExactPeerVersions } from "./peer-version-check.js"; import { resolveSentryDsns } from "./sentry-dsn.js"; @@ -132,6 +133,8 @@ export interface RunFailureEvent { agentAdapter: string; /** The run status that triggered this report. */ runStatus: RunFailureStatus; + /** Bounded, redacted diagnostics selected by the run failure reporter. */ + diagnostics?: RunFailureDiagnostics; /** Recorded process exit evidence, when available. Validated before capture. */ exitCode?: number | null; signal?: string | null; @@ -174,7 +177,33 @@ export function captureRunFailure(event: RunFailureEvent): void { // Sentry's async scope isolation is absent when OTel setup is skipped. // A withScope mutation can then persist into unrelated later captures. // Pass these fields on this event only; do not mutate the ambient scope. - handle.captureException(new Error(event.errorMessage), { + const diagnostics = event.diagnostics; + const contexts: Record> = {}; + if (diagnostics) { + contexts.run_execution = { ...diagnostics.execution, truncatedFields: diagnostics.truncatedFields }; + if (Object.keys(diagnostics.adapter).length) contexts.adapter_failure = diagnostics.adapter; + if (Object.keys(diagnostics.provider).length) contexts.provider_failure = diagnostics.provider; + diagnostics.exceptions.forEach(({ name, code, status, requestId }, index) => { + contexts[`run_exception_${index}`] = { name, code, status, requestId }; + }); + } + // Rebuild only sanitized exception fields. Passing a raw SDK Error can + // serialize its request/response, headers, or other enumerable properties. + let cause: Error | undefined; + for (const entry of [...(diagnostics?.exceptions ?? [])].reverse()) { + const error: Error = new Error(entry.message ?? event.errorMessage, cause ? { cause } : undefined); + error.name = entry.name ?? "Error"; + error.stack = entry.stack; + cause = error; + } + const exception = cause ?? new Error(event.errorMessage); + if (!cause) { + // A saved adapter result is not a thrown Error. Do not pretend that the + // reporter's own stack is the failure location. + exception.stack = typeof diagnostics?.adapter.stackPreview === "string" + ? diagnostics.adapter.stackPreview : undefined; + } + handle.captureException(exception, { tags: { run_id: event.runId, task_id: event.taskId, @@ -183,6 +212,7 @@ export function captureRunFailure(event: RunFailureEvent): void { run_status: event.runStatus, }, contexts: { + ...contexts, run_failure: { taskId: event.taskId, runId: event.runId, diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts new file mode 100644 index 00000000000..838d63f8d01 --- /dev/null +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import type { heartbeatRuns } from "@paperclipai/db"; +import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js"; + +type Run = typeof heartbeatRuns.$inferSelect; +const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides }) as Run; +const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error }); + +describe("run failure diagnostics", () => { + it("selects declared environment secrets under opaque names and common credential keys", () => { + expect(collectRunFailureSecretValues({ + CUSTOM_BINDING: "bound-opaque-value", ACCESS_TOKEN: "plain-opaque-value", REGION: "us-east-1", EMPTY_KEY: "", + }, ["CUSTOM_BINDING"])).toEqual(["bound-opaque-value", "plain-opaque-value", "us-east-1"]); + expect(collectRunFailureSecretValues({ PATH: "/usr/bin", CUSTOM_BINDING: "opaque" }, [], true)).toEqual(["opaque"]); + expect(collectRunFailureSecretValues({ PATH: "secret-override" }, ["PATH"], true)).toEqual(["secret-override"]); + }); + + it("redacts encoded credentials and URL passwords without expanding replacement markers", () => { + const secret = 'opaque/"credential'; + const values = collectRunFailureSecretValues({ DATABASE_URL: `postgres://user:${encodeURIComponent(secret)}@example/db` }); + const result = redactRunFailureSecretValues({ text: `${secret} ${encodeURIComponent(secret)} ${JSON.stringify(secret)}`, status: 503 }, [...values, "REDACTED"]); + expect(result.text).not.toContain("opaque"); + expect(result.status).toBe(503); + expect(result.text).not.toContain("***[REDACTED]"); + }); + + it("preserves known boolean and numeric settings without exempting unknown or declared secrets", () => { + const env = { PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_DB_BACKUP_RETENTION_DAYS: "1" }; + expect(collectRunFailureSecretValues(env, [], true)).toEqual([]); + expect(collectRunFailureSecretValues(env, ["PAPERCLIP_DB_BACKUP_ENABLED"], true)).toEqual(["false"]); + expect(collectRunFailureSecretValues({ CUSTOM_BINDING: "1" }, [], true)).toEqual(["1"]); + expect(collectRunFailureSecretValues({ PAPERCLIP_DB_BACKUP_ENABLED: "opaque" }, [], true)).toEqual(["opaque"]); + }); + it("preserves generic exceptions, numeric codes and nested network causes", () => { + const root = Object.assign(new Error("network failed"), { code: "ECONNRESET", statusCode: 502, request_id: "req-123" }); + const outer = Object.assign(new TypeError("adapter failed", { cause: root }), { code: -32000 }); + const result = sanitizeRunFailureDiagnostics(collect(outer)); + expect(result.exceptions).toMatchObject([ + { name: "TypeError", message: "adapter failed", code: "-32000" }, + { name: "Error", message: "network failed", code: "ECONNRESET", status: 502, requestId: "req-123" }, + ]); + expect(result.exceptions[0].stack).toContain("run-failure-diagnostics.test.ts"); + }); + + it("bounds cyclic, deep, and string causes without inspecting arbitrary error objects", () => { + const cyclic = new Error("cycle"); cyclic.cause = cyclic; + expect(collect(cyclic)).toMatchObject({ exceptions: [{ message: "cycle" }], truncatedFields: ["exceptions.cycle"] }); + let error = new Error("leaf"); + for (let i = 0; i < 10; i++) error = new Error(`level-${i}`, { cause: error }); + expect(collect(error).exceptions).toHaveLength(4); + expect(collect(error).truncatedFields).toContain("exceptions.depth"); + expect(collect(new Error("outer", { cause: "string cause" })).exceptions).toHaveLength(2); + const getter = Object.defineProperty({ message: "valid message" }, "response", { get: () => { throw new Error("must not read"); } }); + expect(collect(getter).exceptions).toEqual([{ message: "valid message" }]); + expect(collect({ arbitrary: "private payload" }).exceptions).toEqual([]); + expect(collect(Object.defineProperty({}, "message", { get: () => { throw new Error("bad getter"); } })).exceptions).toEqual([]); + }); + + it("redacts credential forms before truncation in all diagnostic text", () => { + const failure = collectRunFailureDiagnostics(run({ resultJson: { + terminalSessionFailure: { category: "service", title: 'password="secret-password"', details: 'Authorization: Bearer secret-bearer' }, + } }), { error: Object.assign(new Error('apiKey="secret-api-key"'), { cause: new Error('token="secret-token"') }), adapterErrorMeta: { causeMessage: 'password="secret-password"' } }); + const result = sanitizeRunFailureDiagnostics(failure); + expect(JSON.stringify(result)).not.toContain("secret-"); + expect(sanitizeRunFailureText('x'.repeat(90) + ' password="secret-at-cut-boundary"', 110)).not.toContain("secret-at"); + expect(sanitizeRunFailureText('pass\x1b[31mword="secret-colored"', 200)).not.toContain("secret-colored"); + }); + + it("reports truncation, preserves UTF-16 pairs, and keeps the total payload bounded", () => { + const huge = "😀".repeat(100_000); + let error = Object.assign(new Error(huge), { stack: huge }); + for (let i = 0; i < 5; i++) error = Object.assign(new Error(huge, { cause: error }), { stack: huge }); + const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: { + terminalSessionFailure: { category: "service", title: huge, details: huge, truncatedFields: ["details", "private-field"] }, + } }), { error })); + expect(String(result.provider.title).length).toBeLessThanOrEqual(4096); + expect(String(result.provider.details).length).toBeLessThanOrEqual(12288); + expect(JSON.stringify(result)).not.toMatch(/[\ud800-\udfff]/u); + expect(result.truncatedFields).toContain("provider.details"); + expect(result.truncatedFields).toContain("exceptions.0.stack"); + expect(result.truncatedFields).not.toContain("provider.private-field"); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(150_000); + }); + + it("keeps scalar native and legacy execution evidence and rejects non-finite durations", () => { + const result = collectRunFailureDiagnostics(run({ + runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", + startedAt: new Date(1000), finishedAt: new Date(8500), + resultJson: { stopReason: "failed", timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream", summary: "private prose" }, + }), { phase: "execute" }); + expect(result.execution).toMatchObject({ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", durationMs: 7500, timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream" }); + expect(JSON.stringify(result)).not.toContain("private prose"); + expect(collectRunFailureDiagnostics(run({ startedAt: new Date(NaN), finishedAt: new Date() }), {}).execution).not.toHaveProperty("durationMs"); + }); +}); diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 9b32245e763..2b7e184e484 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import os from "node:os"; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, @@ -9,6 +9,11 @@ import { const mockCaptureRunFailure = vi.hoisted(() => vi.fn()); const mockRedactCurrentUserText = vi.hoisted(() => vi.fn()); +const mockResolveSecret = vi.hoisted(() => vi.fn()); + +vi.mock("../../secrets/provider-registry.js", () => ({ + getSecretProvider: () => ({ resolveVersion: mockResolveSecret }), +})); vi.mock("../../sentry.js", () => ({ captureRunFailure: mockCaptureRunFailure, @@ -23,8 +28,7 @@ vi.mock("../../log-redaction.js", async (importOriginal) => { }); import { reportRunFailure, waitForPendingRunFailureReports } from "../run-failure-report.js"; -import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../../redaction.js"; -import { redactCurrentUserText } from "../../log-redaction.js"; +import { REDACTED_EVENT_VALUE } from "../../redaction.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -34,14 +38,29 @@ describeEmbeddedPostgres("reportRunFailure", () => { let tempDb: Awaited> | null = null; let companyId!: string; let agentId!: string; + let inheritedEnv: NodeJS.ProcessEnv; beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-report-"); db = createDb(tempDb.connectionString); }, 20_000); + beforeEach(() => { + // Unknown host values intentionally count as secrets, including short values + // such as "1". Keep these fixtures independent of the developer/CI environment + // and add secret values explicitly in the tests that exercise redaction. + inheritedEnv = process.env; + process.env = Object.fromEntries( + ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"] + .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]), + ); + }); + afterEach(async () => { vi.clearAllMocks(); + vi.unstubAllEnvs(); + process.env = inheritedEnv; + await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companies); }); @@ -141,6 +160,98 @@ describeEmbeddedPostgres("reportRunFailure", () => { } }); + it("reports selected provider diagnostics and the original cause without copying arbitrary payloads", async () => { + await seedCompanyAndAgent(); + const cause = Object.assign(new Error("provider unreachable"), { + code: "ECONNRESET", requestId: "request-123", status: 503, + response: { body: "private-response" }, + }); + const error = new Error("adapter threw", { cause }); + const run = buildRun({ + runtimeMode: "legacy", + executionStage: "execute", + error: "adapter threw", + stderrExcerpt: "private-stderr", + stdoutExcerpt: "private-stdout", + contextSnapshot: { prompt: "private-prompt" }, + resultJson: { + terminalSessionFailure: { category: "service", title: "Provider unavailable", details: "request-123 failed", raw: "private-provider-raw" }, + timeoutFired: false, summary: "private-summary", env: { SECRET: "private-env" }, + }, + }); + await reportRunFailure(db, run, { error, phase: "execute", adapterErrorMeta: { phase: "turn", retryable: true, response: "private-adapter-response" } }); + const captured = mockCaptureRunFailure.mock.calls[0][0]; + expect(captured.diagnostics).toMatchObject({ + execution: { runtimeMode: "legacy", executionStage: "execute", failurePhase: "execute", timeoutFired: false }, + adapter: { phase: "turn", retryable: true }, + provider: { category: "service", title: "Provider unavailable", details: "request-123 failed" }, + exceptions: [{ message: "adapter threw" }, { message: "provider unreachable", code: "ECONNRESET", status: 503, requestId: "request-123" }], + }); + expect(captured.diagnostics.exceptions[0].stack).toContain("run-failure-report.test.ts"); + expect(JSON.stringify(captured)).not.toContain("private-"); + expect(error.cause).toBe(cause); + }); + + it.each([false, true])("redacts registered run secrets and fails closed when resolution fails: %s", async (fails) => { + await seedCompanyAndAgent(); + const secret = "opaque-registered-value"; + const run = buildRun({ + error: `connection failed: ${secret}`, + contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "fixture", material: { encrypted: "fixture" } }] }, + resultJson: { terminalSessionFailure: { category: "service", details: `upstream rejected ${secret}` } }, + }); + await db.insert(heartbeatRuns).values(run); + if (fails) mockResolveSecret.mockRejectedValueOnce(new Error("fixture resolution failed")); + else mockResolveSecret.mockResolvedValueOnce(secret); + + await expect(reportRunFailure(db, run, { + error: new Error(`failed ${secret}`, { cause: new Error(`cause ${secret}`) }), + adapterErrorMeta: { causeMessage: `adapter ${secret}` }, + })).resolves.toBeUndefined(); + expect(mockResolveSecret).toHaveBeenCalledTimes(1); + if (fails) { + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + } else { + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + const captured = mockCaptureRunFailure.mock.calls[0][0]; + expect(JSON.stringify(captured)).not.toContain(secret); + expect(captured.diagnostics.provider.details).toContain(REDACTED_EVENT_VALUE); + expect(captured.diagnostics.exceptions[1].message).toContain(REDACTED_EVENT_VALUE); + } + }); + + it("redacts runtime and host environment secret values even without a persisted registry", async () => { + await seedCompanyAndAgent(); + const runtimeSecret = "opaque-runtime-value"; + const hostSecret = "opaque-host-value"; + vi.stubEnv("FIXTURE_CUSTOM_VALUE", hostSecret); + const text = `connection failed: ${runtimeSecret} ${hostSecret}`; + const run = buildRun({ error: text, contextSnapshot: null, + resultJson: { terminalSessionFailure: { details: text } }, + }); + await reportRunFailure(db, run, { + error: new Error(text, { cause: new Error(text) }), + adapterErrorMeta: { causeMessage: text, stackPreview: text }, + secretValues: [runtimeSecret], + }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + const captured = mockCaptureRunFailure.mock.calls[0][0]; + expect(JSON.stringify(captured)).not.toContain(runtimeSecret); + expect(JSON.stringify(captured)).not.toContain(hostSecret); + expect(captured.diagnostics.exceptions).toHaveLength(2); + expect(captured).not.toHaveProperty("secretValues"); + }); + + it("still redacts short values from unknown host settings", async () => { + await seedCompanyAndAgent(); + vi.stubEnv("FIXTURE_CUSTOM_VALUE", "1"); + + await reportRunFailure(db, buildRun()); + + expect(mockCaptureRunFailure.mock.calls[0][0].errorMessage) + .toBe(`the provider process exited with code ${REDACTED_EVENT_VALUE}`); + }); + it("captures nothing for succeeded, cancelled, and interrupted", async () => { await seedCompanyAndAgent(); for (const status of ["succeeded", "cancelled", "interrupted"] as const) { @@ -259,9 +370,10 @@ describeEmbeddedPostgres("reportRunFailure", () => { await reportRunFailure(db, run); - const expectedErrorMessage = redactSensitiveText(redactCurrentUserText(rawError)); const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0]; - expect(errorMessage).toBe(expectedErrorMessage); + // Environment-value redaction can fully mask the username before the + // current-user redactor applies its partial mask. Both remove the home path. + expect(errorMessage).toContain("workspace/report.log"); expect(errorMessage).not.toContain(homeDir); }); @@ -276,10 +388,13 @@ describeEmbeddedPostgres("reportRunFailure", () => { const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0]; expect(errorMessage).toHaveLength(MAX_ERROR_MESSAGE_LENGTH); - expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH)); + expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH - 12) + "\n[truncated]"); }); - it("does not change a short error message", async () => { + it("preserves a short error message with known public host settings", async () => { + vi.stubEnv("PAPERCLIP_DB_BACKUP_ENABLED", "false"); + vi.stubEnv("PAPERCLIP_DB_BACKUP_RETENTION_DAYS", "1"); + vi.stubEnv("GITHUB_RUN_ATTEMPT", "1"); await seedCompanyAndAgent(); const shortError = "the provider process exited with code 1"; const run = buildRun({ status: "failed", error: shortError }); @@ -326,7 +441,7 @@ describeEmbeddedPostgres("reportRunFailure", () => { const { errorCode } = mockCaptureRunFailure.mock.calls[0][0]; expect(errorCode).toHaveLength(MAX_ERROR_CODE_LENGTH); - expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH)); + expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH - 12) + "\n[truncated]"); }); it("sends errorCode null unchanged when the run holds no error code", async () => { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 2f86e242fec..490101f2fd0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -326,6 +326,7 @@ import { emitAgentTaskRunById, } from "./agent-task-run-telemetry.js"; import { reportRunFailure } from "./run-failure-report.js"; +import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; import { budgetService, type BudgetEnforcementScope } from "./budgets.js"; import { secretService, type MissingRuntimeBinding } from "./secrets.js"; @@ -12754,12 +12755,13 @@ export function heartbeatService( function emitTerminalAgentTaskRun( updated: typeof heartbeatRuns.$inferSelect, previousStatus: string | null, + failureReport?: RunFailureReportOptions, ) { if (!isHeartbeatRunTerminalStatus(updated.status)) return; if (previousStatus === updated.status) return; clearHeartbeatRunRuntimeStatus(updated.id); void emitAgentTaskRun(db, updated); - void reportRunFailure(db, updated); + void reportRunFailure(db, updated, failureReport); } async function setRunStatus( @@ -12833,8 +12835,9 @@ export function heartbeatService( runId: string, status: string, patch?: Partial, + failureReport?: RunFailureReportOptions, ) { - return setRunStatusFromLive(runId, status, ["running"], patch); + return setRunStatusFromLive(runId, status, ["running"], patch, failureReport); } // Move a run to a new status only when its current status is one of @@ -12847,6 +12850,7 @@ export function heartbeatService( status: string, fromStatuses: string[], patch?: Partial, + failureReport?: RunFailureReportOptions, ) { // fromStatuses can name a terminal status as its own source (for example, // an idempotent "still failed" patch), so the write below is not always a @@ -12916,7 +12920,7 @@ export function heartbeatService( payload: buildHeartbeatRunStatusLiveEventPayload(updated), }); publishRunLifecyclePluginEvent(updated); - emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null); + emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport); return { run: updated, updated: true as const }; } @@ -20238,6 +20242,7 @@ export function heartbeatService( ReturnType > | null = null; let providerTraceFinalized = false; + let readFailureReportSecrets: () => string[] = () => []; try { const agent = await getAgent(run.agentId); @@ -21414,6 +21419,7 @@ export function heartbeatService( secretsSvc, trustPreset, }); + readFailureReportSecrets = () => collectRunFailureSecretValues(resolvedConfig.env, secretKeys); if (aiBinding) { try { managedAiRuntime = await prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig }); @@ -21504,6 +21510,11 @@ export function heartbeatService( // Always replace this runtime-only field; caller wake data cannot supply skills. context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions }; let runtimeConfig: Record = connectorDelivery.config; + const resolvedFailureSecrets = readFailureReportSecrets(); + readFailureReportSecrets = () => [ + ...resolvedFailureSecrets, + ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys), + ]; const latestAgentConfigRevision = await getLatestAgentConfigRevision( agent.companyId, agent.id, @@ -25191,6 +25202,7 @@ export function heartbeatService( run.id, status, finalRunPatch, + { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() }, ); let persistedRun: typeof heartbeatRuns.$inferSelect | null = persistedRunWrite.run; @@ -25830,7 +25842,7 @@ export function heartbeatService( logBytes: logSummary?.bytes, logSha256: logSummary?.sha256, logCompressed: logSummary?.compressed ?? false, - }); + }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() }); if ( !failedRunWrite.updated && !( @@ -26084,7 +26096,7 @@ export function heartbeatService( : setupFailureResultJson ? { resultJson: setupFailureResultJson } : {}), - }).catch(() => ({ run: null, updated: false as const })); + }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const })); if (!setupFailureWrite.updated) { logger.info( { diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts new file mode 100644 index 00000000000..86197fc1a1d --- /dev/null +++ b/server/src/services/run-failure-diagnostics.ts @@ -0,0 +1,227 @@ +import type { heartbeatRuns } from "@paperclipai/db"; +import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction"; +import { redactCurrentUserText } from "../log-redaction.js"; +import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js"; + +type Run = typeof heartbeatRuns.$inferSelect; +type Context = Record; + +export interface RunFailureReportOptions { + /** The caught exception, before callers flatten it to a message. */ + error?: unknown; + /** Structured adapter error metadata; only known diagnostic keys are read. */ + adapterErrorMeta?: unknown; + phase?: "setup" | "execute"; + /** Runtime-only values to redact; never included in the captured event. */ + secretValues?: readonly string[]; +} + +export interface RunFailureException { + name?: string; + message?: string; + stack?: string; + code?: string; + status?: number; + requestId?: string; +} + +export interface RunFailureDiagnostics { + execution: Context; + adapter: Context; + provider: Context; + exceptions: RunFailureException[]; + truncatedFields: string[]; +} + +function read(value: unknown, key: string): unknown { + if (!value || typeof value !== "object") return undefined; + try { + return (value as Record)[key]; + } catch { + return undefined; + } +} + +function scalars(value: unknown, fields: readonly string[]): Context { + const result: Context = {}; + for (const field of fields) { + const entry = read(value, field); + if ( + typeof entry === "string" || typeof entry === "boolean" || + (typeof entry === "number" && Number.isFinite(entry)) + ) result[field] = entry; + } + return result; +} + +// Unknown inherited values may contain credentials under arbitrary names. +// Explicitly configured values remain private, including overrides of these keys. +const PUBLIC_ENV_KEYS = new Set([ + "PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "HOME", "USERPROFILE", + "HOMEDRIVE", "HOMEPATH", "USER", "USERNAME", "LOGNAME", "SHELL", "LANG", + "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR", "TEMP", "TMP", "NODE_ENV", + "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", + "PWD", "OLDPWD", "INIT_CWD", "SHLVL", "_", "TERM", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", "COLORTERM", + "NPM_LIFECYCLE_EVENT", "NPM_COMMAND", "NPM_EXEC_PATH", "NPM_EXECPATH", "NPM_NODE_EXECPATH", + "NPM_PACKAGE_NAME", "NPM_PACKAGE_VERSION", "NPM_PACKAGE_JSON", "NPM_CONFIG_USER_AGENT", + "XPC_SERVICE_NAME", "VITEST_POOL_ID", "VITEST_WORKER_ID", + "PAPERCLIP_AGENT_ID", "PAPERCLIP_COMPANY_ID", "PAPERCLIP_RUN_ID", "PAPERCLIP_TASK_ID", +]); +const PUBLIC_BOOLEAN_ENV_KEYS = new Set([ + "OPENCODE_ALLOW_ALL_MODELS", "CLAUDE_CODE_USE_BEDROCK", "GOOGLE_GENAI_USE_GCA", + "CI", "NO_COLOR", "FORCE_COLOR", + "VITEST", "MALLOCNANOZONE", "CODEX_CI", "CODEX_SHELL", "CODEX_SAGE_BACKFILL_TRACKER_TAB_REUSE", + "DEV", "PROD", "SSR", "PAPERCLIP_REQUIRE_SENTRY_TEST_SDK", + "PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION", + "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "PAPERCLIP_SECRETS_STRICT_MODE", + "SERVE_UI", "PAPERCLIP_UI_DEV_MIDDLEWARE", "HEARTBEAT_SCHEDULER_ENABLED", "PAPERCLIP_ANNOUNCEMENTS_ENABLED", + "GITHUB_ACTIONS", "GITHUB_REF_PROTECTED", "RUNNER_DEBUG", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG", +]); +const PUBLIC_NUMBER_ENV_KEYS = new Set([ + "PORT", "PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES", "PAPERCLIP_DB_BACKUP_RETENTION_DAYS", + "PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "HEARTBEAT_SCHEDULER_INTERVAL_MS", + "GITHUB_RUN_ATTEMPT", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ID", "GITHUB_RETENTION_DAYS", +]); + +/** Include declared bindings and unknown values, not just credential-like keys. */ +export function collectRunFailureSecretValues( + env: unknown, secretKeys: Iterable = [], inherited = false, +): string[] { + if (!env || typeof env !== "object") return []; + const strings = Object.fromEntries(Object.entries(env).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + )); + const declared = new Set(secretKeys); + const values: string[] = []; + for (const [key, value] of Object.entries(strings)) { + if (!value) continue; + const publicSetting = (inherited && ( + PUBLIC_ENV_KEYS.has(key.toUpperCase()) || + (PUBLIC_NUMBER_ENV_KEYS.has(key.toUpperCase()) && /^\d+$/.test(value)) || + (key === "BASE_URL" && value === "/") || + (key === "MODE" && /^(?:test|development|production)$/.test(value)) + )) || + (PUBLIC_BOOLEAN_ENV_KEYS.has(key.toUpperCase()) && /^(?:0|1|true|false)$/i.test(value)); + if (declared.has(key) || !publicSetting) values.push(value); + try { + const password = new URL(value).password; + if (password) values.push(value, password, decodeURIComponent(password)); + } catch { /* ordinary environment values are not URLs */ } + } + return [...new Set(values)].sort((a, b) => b.length - a.length); +} + +/** Redact literal, JSON-escaped and URI-encoded forms without rewriting markers. */ +export function redactRunFailureSecretValues(input: T, values: readonly string[]): T { + const forms = [...new Set(values.filter(Boolean).flatMap((value) => { + const forms = [value, JSON.stringify(value).slice(1, -1)]; + try { forms.push(encodeURIComponent(value)); } catch { /* malformed Unicode */ } + return forms; + }))].sort((a, b) => b.length - a.length); + if (forms.length === 0) return input; + const pattern = new RegExp(forms.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "gu"); + const redact = (value: unknown): unknown => { + if (typeof value === "string") return value.replace(pattern, () => REDACTED_EVENT_VALUE); + if (Array.isArray(value)) return value.map(redact); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redact(entry)])); + } + return value; + }; + return redact(input) as T; +} + +/** Snapshot only diagnostics. Never walk a request, response, config or prompt. */ +export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportOptions): RunFailureDiagnostics { + const execution = scalars(run, [ + "runtimeMode", "executionStage", "nativePhase", "driverKind", "driverVersion", + ]); + if (options.phase) execution.failurePhase = options.phase; + if (run.startedAt && run.finishedAt) { + const durationMs = run.finishedAt.getTime() - run.startedAt.getTime(); + if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; + } + const result = run.resultJson; + Object.assign(execution, scalars(result, [ + "mode", "stopReason", "timeoutFired", "timeoutSource", "timeoutConfigured", + "effectiveTimeoutSec", "errorFamily", + ])); + const adapter = scalars(options.adapterErrorMeta, [ + "category", "phase", "errorName", "acpCode", "causeMessage", "retryable", + "stackPreview", "status", "statusCode", "requestId", + ]); + const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]); + const truncatedFields: string[] = []; + const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields"); + if (Array.isArray(providerTruncation)) { + for (const field of ["title", "details"]) { + if (providerTruncation.includes(field)) truncatedFields.push(`provider.${field}`); + } + } + const exceptions: RunFailureException[] = []; + const seen = new Set(); + let error = options.error; + while (error !== undefined && error !== null && exceptions.length < 4) { + if (seen.has(error)) { + truncatedFields.push("exceptions.cycle"); + break; + } + seen.add(error); + if (typeof error === "string") { + exceptions.push({ message: error }); + error = undefined; + break; + } + if (typeof error !== "object") break; + const entry: RunFailureException = {}; + for (const field of ["name", "message", "stack", "code"] as const) { + const value = read(error, field); + if (typeof value === "string") entry[field] = value; + else if (field === "code" && typeof value === "number" && Number.isFinite(value)) entry.code = String(value); + } + const status = read(error, "status") ?? read(error, "statusCode"); + if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) entry.status = status; + const requestId = read(error, "requestId") ?? read(error, "request_id"); + if (typeof requestId === "string") entry.requestId = requestId; + if (Object.keys(entry).length > 0) exceptions.push(entry); + else break; + error = read(error, "cause"); + } + if (error !== undefined && error !== null && exceptions.length === 4) truncatedFields.push("exceptions.depth"); + return { execution, adapter, provider, exceptions, truncatedFields }; +} + +/** Redact complete values before cutting, including credentials across a cut. */ +export function sanitizeRunFailureText(input: string, limit: number): string { + const normalized = input + .replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "") + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "") + .replace(/[\ud800-\udfff]/gu, "\ufffd"); + const clean = redactSensitiveText(redactDiagnosticText(redactCurrentUserText(normalized))); + if (clean.length <= limit) return clean; + const suffix = "\n[truncated]"; + let end = limit - suffix.length; + if ((clean.codePointAt(end - 1) ?? 0) > 0xffff) end--; + return clean.slice(0, end) + suffix; +} + +/** Called after the run's registered secret values have also been removed. */ +export function sanitizeRunFailureDiagnostics(raw: RunFailureDiagnostics): RunFailureDiagnostics { + const truncatedFields = [...raw.truncatedFields]; + const text = (value: string, path: string, limit: number) => { + const sanitized = sanitizeRunFailureText(value, limit); + if (sanitized.endsWith("\n[truncated]")) truncatedFields.push(path); + return sanitized; + }; + const context = (value: Context, prefix: string, limits: Record = {}): Context => + Object.fromEntries(Object.entries(value).map(([key, value]) => [key, + typeof value === "string" ? text(value, `${prefix}.${key}`, limits[key] ?? 200) : value, + ])); + return { + execution: context(raw.execution, "execution"), + adapter: context(raw.adapter, "adapter", { causeMessage: 2048, stackPreview: 8192 }), + provider: context(raw.provider, "provider", { title: 4096, details: 12288 }), + exceptions: raw.exceptions.map((entry, i) => context(entry as Context, `exceptions.${i}`, { message: 2048, stack: 8192 })), + truncatedFields: [...new Set(truncatedFields)], + }; +} diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 7e403455138..0c044b66150 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -1,8 +1,14 @@ -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { captureRunFailure, type RunFailureStatus } from "../sentry.js"; -import { redactCurrentUserText } from "../log-redaction.js"; -import { redactSensitiveText } from "../redaction.js"; +import { + collectRunFailureDiagnostics, + collectRunFailureSecretValues, + redactRunFailureSecretValues, + sanitizeRunFailureDiagnostics, + sanitizeRunFailureText, + type RunFailureReportOptions, +} from "./run-failure-diagnostics.js"; import { logger } from "../middleware/logger.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -23,15 +29,6 @@ const pendingRunFailureReports = new Set>(); /** Bounds the shutdown wait, so one stuck report cannot hang the process exit. */ const PENDING_REPORT_DRAIN_TIMEOUT_MS = 5_000; -/** - * Remove a credential and the current user's home path from adapter-supplied - * text, then cut the result to `maxLength`. Run the length cut after the - * redaction, so a credential cannot survive at a cut boundary. - */ -function sanitizeAdapterText(input: string, maxLength: number): string { - return redactSensitiveText(redactCurrentUserText(input)).slice(0, maxLength); -} - function isRunFailureStatus(status: string): status is RunFailureStatus { return status === "failed" || status === "timed_out"; } @@ -53,10 +50,10 @@ function readTaskId(run: HeartbeatRun): string | null { * its own in-flight promise, so a caller that does not await it still lets * shutdown find and wait for the report — see `waitForPendingRunFailureReports`. */ -export function reportRunFailure(db: Db, run: HeartbeatRun): Promise { +export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureReportOptions = {}): Promise { if (!isRunFailureStatus(run.status)) return Promise.resolve(); const runStatus = run.status; - const report = captureTerminalRunFailure(db, run, runStatus); + const report = captureTerminalRunFailure(db, run, runStatus, options); pendingRunFailureReports.add(report); void report.finally(() => pendingRunFailureReports.delete(report)); return report; @@ -66,12 +63,21 @@ async function captureTerminalRunFailure( db: Db, run: HeartbeatRun, runStatus: RunFailureStatus, + options: RunFailureReportOptions, ): Promise { try { + const snapshot = redactRunFailureSecretValues({ + errorMessage: run.error ?? "", + errorCode: run.errorCode ?? null, + diagnostics: collectRunFailureDiagnostics(run, options), + }, [...new Set([ + ...collectRunFailureSecretValues(process.env, [], true), + ...(options.secretValues ?? []), + ])].sort((a, b) => b.length - a.length)); const agent = await db .select({ adapterType: agents.adapterType }) .from(agents) - .where(eq(agents.id, run.agentId)) + .where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId))) .then((rows) => rows[0] ?? null); const taskId = readTaskId(run); @@ -80,18 +86,26 @@ async function captureTerminalRunFailure( return; } + // Resolve registered values before truncation. A failed resolution must + // not send an incompletely redacted report. + let redacted = snapshot; + if (Array.isArray(run.contextSnapshot?.paperclipSecretRedactions)) { + const { createRunSecretRedactionRegistry } = await import("./run-secret-redaction.js"); + redacted = await createRunSecretRedactionRegistry(db).redactForRun(run.companyId, run.id, snapshot); + } captureRunFailure({ taskId, runId: run.id, - errorMessage: sanitizeAdapterText(run.error ?? "", MAX_ERROR_MESSAGE_LENGTH), + errorMessage: sanitizeRunFailureText(redacted.errorMessage, MAX_ERROR_MESSAGE_LENGTH), errorCode: - run.errorCode === null + redacted.errorCode === null ? null - : sanitizeAdapterText(run.errorCode, MAX_ERROR_CODE_LENGTH), + : sanitizeRunFailureText(redacted.errorCode, MAX_ERROR_CODE_LENGTH), agentAdapter: agent?.adapterType ?? UNKNOWN_ADAPTER, runStatus, exitCode: run.exitCode, signal: run.signal, + diagnostics: sanitizeRunFailureDiagnostics(redacted.diagnostics), }); } catch (err) { logger.warn({ err, runId: run.id }, "failed to report run failure to Sentry");