diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 217e785025a..36a413d3ff8 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -1002,3 +1002,24 @@ the bounded stall policy, and operator recovery requirements stay unchanged. A later successful continuation does not establish the cause of an earlier failure that recorded only `Error`. + +### Failure cause of generic run failures + +A run that fails with a generic code (`adapter_failed`, `setup_failed`) keeps +that code, because callers compare against it, and records +`resultJson.failureCause`: one identifier of at most 80 characters +(`[A-Za-z0-9_.:-]`). For a thrown error it is the provider's HTTP status +(`provider_http_429`) or category, else the first safe code in the error and up +to three nested `cause` objects (`40001`, `ECONNRESET`, the same extraction as +the Runner's event commit stall diagnostics), else the error's class name. For +an adapter result without a code it is the error family, `signal_`, +`process_cancellation_failed`, `exit_code_` or `adapter_reported_error`. It +never copies message text. The server's failure log line carries the same +cause, and the board's run list shows it, for example "adapter failed (40001)". +Older rows have no cause and read as before. + +Run error text keeps SQL statements off the board: a Drizzle `Failed query:` +statement and its parameters, or unwrapped statement text with `$n` +placeholders, is replaced with "Database query failed ()". The wrapper +text before the statement is kept. + diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index f1d2597bad1..90430348153 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -1,3 +1,4 @@ +import { safeErrorCauseCode } from "./safe-error-cause.js"; import { AGENT_OBSERVABILITY_ENV_KEYS } from "../drivers/agent-observability.js"; import { configuredEnvironment } from "../configured-environment.js"; import { spawn } from "node:child_process"; @@ -878,19 +879,6 @@ function eventSchemaIssuePaths(issues: readonly { path?: unknown }[] | undefined return paths.length ? paths.slice(0, 3).join(",") : "unknown"; } -function eventCommitFailureCause(error: unknown): string { - // Database and transport adapters wrap their coded failure in Error.cause. - // Read only bounded codes: messages can contain SQL, arguments or secrets. - let current = error; - for (let depth = 0; depth < 4 && isRecord(current); depth++) { - const code = current.code; - if (typeof code === "string" && /^[A-Za-z0-9_.:-]{1,80}$/.test(code)) return code; - current = current.cause; - } - const name = error instanceof Error ? error.name : typeof error; - return /^[A-Za-z0-9_.:-]{1,80}$/.test(name) ? name : "unknown_error"; -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -2239,7 +2227,7 @@ export class DurablePrpControlPlane { .then(() => connection.sendJson(frame)) .catch(() => { if (checkpointFailure !== null && this.#protocolIntegrityError === null) { - this.#eventCommitFailed(connection, `checkpoint_flush_failed:${eventCommitFailureCause(checkpointFailure)}`); + this.#eventCommitFailed(connection, `checkpoint_flush_failed:${safeErrorCauseCode(checkpointFailure)}`); } else connection.close(); }); this.#sendQueues.set(connection, sending); @@ -3836,7 +3824,7 @@ export class DurablePrpControlPlane { try { this.#store.save(); } catch (error) { - this.#failEventCommit(connection, `journal_write_failed:${eventCommitFailureCause(error)}`); + this.#failEventCommit(connection, `journal_write_failed:${safeErrorCauseCode(error)}`); return; } this.#sendJson(connection, { @@ -4330,7 +4318,7 @@ export class DurablePrpControlPlane { if (error instanceof NativeSessionProtocolIntegrityError) { this.#failProtocolIntegrity(connection, error); } else { - this.#eventCommitFailed(connection, eventCommitFailureCause(error), sourceSeq); + this.#eventCommitFailed(connection, safeErrorCauseCode(error), sourceSeq); } return; } @@ -4383,7 +4371,7 @@ export class DurablePrpControlPlane { try { this.#store.commit(candidate, !isStreamingEvent(event)); } catch (error) { - this.#failEventCommit(connection, `journal_write_failed:${eventCommitFailureCause(error)}`, sourceSeq); + this.#failEventCommit(connection, `journal_write_failed:${safeErrorCauseCode(error)}`, sourceSeq); return; } } else { @@ -4391,7 +4379,7 @@ export class DurablePrpControlPlane { try { committed = await this.#store.commitEvent(buildCandidate, isCurrentOwner, !isStreamingEvent(event)); } catch (error) { - if (isCurrentOwner()) this.#failEventCommit(connection, `journal_write_failed:${eventCommitFailureCause(error)}`, sourceSeq); + if (isCurrentOwner()) this.#failEventCommit(connection, `journal_write_failed:${safeErrorCauseCode(error)}`, sourceSeq); else connection.close(); return; } @@ -4422,7 +4410,7 @@ export class DurablePrpControlPlane { await this.flushCheckpoint(); } catch (error) { if (this.#eventOwner === connection && this.#identity === identity) { - this.#eventCommitFailed(connection, `checkpoint_flush_failed:${eventCommitFailureCause(error)}`, sourceSeq); + this.#eventCommitFailed(connection, `checkpoint_flush_failed:${safeErrorCauseCode(error)}`, sourceSeq); } return; } diff --git a/packages/paperclip-runner/src/control-plane/safe-error-cause.ts b/packages/paperclip-runner/src/control-plane/safe-error-cause.ts new file mode 100644 index 00000000000..f95ad40e5b7 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/safe-error-cause.ts @@ -0,0 +1,24 @@ +/** A bounded identifier: an error code such as `40001` or `ECONNRESET`, or a class name. */ +export const SAFE_ERROR_CAUSE_PATTERN = /^[A-Za-z0-9_.:-]{1,80}$/; + +/** + * The first safe error code in an error and up to three nested `cause` + * objects, else the outer error's class name, else `unknown_error`. + * + * Database and transport adapters wrap their coded failure in Error.cause. + * Read only bounded codes: messages can contain SQL, arguments or secrets. + */ +export function safeErrorCauseCode(error: unknown): string { + let current = error; + for (let depth = 0; depth < 4 && isRecord(current); depth++) { + const code = current.code; + if (typeof code === "string" && SAFE_ERROR_CAUSE_PATTERN.test(code)) return code; + current = current.cause; + } + const name = error instanceof Error ? error.name : typeof error; + return SAFE_ERROR_CAUSE_PATTERN.test(name) ? name : "unknown_error"; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 6923aae0041..ff4c58df2be 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -136,3 +136,4 @@ export { archiveCursorCloudSessionCheckpoint } from "./drivers/cursor-cloud/chec export { readProviderFailure, type ProviderFailure } from "./contracts/provider-failure.generated.js"; export { providerFailureFromError } from "./drivers/provider-failure.js"; +export { SAFE_ERROR_CAUSE_PATTERN, safeErrorCauseCode } from "./control-plane/safe-error-cause.js"; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 869a8bb364a..7ce86fb2ce0 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2909,6 +2909,7 @@ export type { BoardBootstrap, BoardBootstrapSession } from "./types/board-bootst export { readBoardBootstrap } from "./board-bootstrap.js"; export * from "./provider-rate-limits.js"; export * from "./agent-interaction-permissions.js"; +export * from "./run-failure-cause.js"; export { TASK_IDENTITY_CHANGED_FIELD, issueActivityChangesTaskIdentity, diff --git a/packages/shared/src/run-failure-cause.test.ts b/packages/shared/src/run-failure-cause.test.ts new file mode 100644 index 00000000000..68a726ac3e3 --- /dev/null +++ b/packages/shared/src/run-failure-cause.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText } from "./run-failure-cause.js"; + +describe("run failure cause contract", () => { + it("accepts bounded identifier codes", () => { + for (const code of ["40001", "ECONNRESET", "provider_http_429", "TypeError", "signal_SIGKILL", "a".repeat(RUN_FAILURE_CAUSE_MAX_LENGTH)]) { + expect(normalizeRunFailureCause(code)).toBe(code); + } + }); + + it("rejects message text, SQL, paths, overlong values and non-strings", () => { + for (const value of ["", "select * from users", "Failed query: update x", "/Users/me/project", "a b", + "a".repeat(RUN_FAILURE_CAUSE_MAX_LENGTH + 1), 40001, null, undefined, {}]) { + expect(normalizeRunFailureCause(value)).toBeNull(); + } + }); + + it("reads the cause from a stored result and treats legacy rows as having none", () => { + expect(readRunFailureCause({ failureCause: "40001", stopReason: "adapter_failed" })).toBe("40001"); + expect(readRunFailureCause({ stopReason: "adapter_failed" })).toBeNull(); + expect(readRunFailureCause(null)).toBeNull(); + expect(readRunFailureCause(["40001"])).toBeNull(); + }); + + it("drops Drizzle statement text and parameters but keeps the wrapper and the code", () => { + const message = 'Failed to save usage: Failed query: update "heartbeat_runs" set "usage_json" = $1 where "id" = $2\nparams: {"token":"sk-secret"},run-1'; + const safe = withoutSqlStatementText(message, "40001"); + expect(safe).toBe("Failed to save usage: Database query failed (40001)"); + expect(safe).not.toMatch(/heartbeat_runs|params|sk-secret|\$1/); + expect(withoutSqlStatementText("Failed query: select 1", null)).toBe("Database query failed"); + }); + + it("drops unwrapped statements with placeholders and ignores an invalid cause", () => { + expect(withoutSqlStatementText('Lookup failed: select "id" from "issues" where "id" = $1', "not a code")) + .toBe("Lookup failed: Database query failed"); + expect(withoutSqlStatementText("insert into agents (name) values ($1)")).toBe("Database query failed"); + }); + + it("leaves ordinary error text unchanged", () => { + for (const message of ["Adapter failed", "Select a model from the list", "Process exited with code 1", "Costs $5 per run"]) { + expect(withoutSqlStatementText(message, "40001")).toBe(message); + } + }); +}); diff --git a/packages/shared/src/run-failure-cause.ts b/packages/shared/src/run-failure-cause.ts new file mode 100644 index 00000000000..6f7c646d715 --- /dev/null +++ b/packages/shared/src/run-failure-cause.ts @@ -0,0 +1,45 @@ +/** + * `resultJson.failureCause` on a failed heartbeat run: a short code that says + * why a run with a generic error code (such as `adapter_failed`) failed, for + * example a SQLSTATE (`40001`), a transport code (`ECONNRESET`), a provider + * status (`provider_http_429`) or an error class name. + * + * The shape is closed and bounded: one identifier token of at most 80 + * characters. It never carries message text, SQL, prompts, paths or secrets. + * Rows written before the field existed have no cause and read as `null`. + */ +export const RUN_FAILURE_CAUSE_MAX_LENGTH = 80; +export const RUN_FAILURE_CAUSE_PATTERN = /^[A-Za-z0-9_.:-]{1,80}$/; + +/** The stored cause when it has the closed shape, else `null` (absent, legacy or malformed). */ +export function normalizeRunFailureCause(value: unknown): string | null { + return typeof value === "string" && RUN_FAILURE_CAUSE_PATTERN.test(value) ? value : null; +} + +/** `resultJson.failureCause` from a run's stored result, validated. */ +export function readRunFailureCause(resultJson: unknown): string | null { + if (typeof resultJson !== "object" || resultJson === null || Array.isArray(resultJson)) return null; + return normalizeRunFailureCause((resultJson as Record).failureCause); +} + +// Drizzle wraps a failed statement as "Failed query: \nparams: ". +const FAILED_QUERY_MARKER = /\bFailed query:/i; +// Unwrapped statement text: a parameter placeholder plus a statement shape. +const SQL_PLACEHOLDER = /\$\d+\b/; +const SQL_STATEMENT = /\b(?:select\b[\s\S]*?\bfrom\b|insert\s+into\b|update\b[\s\S]*?\bset\b|delete\s+from\b|with\b[\s\S]*?\bas\s*\()/i; + +/** + * Run error text for the board with any SQL statement and its parameters + * removed. Text before the statement (the caller's own wrapper) is kept and the + * statement is replaced by "Database query failed ()". Text without SQL + * is returned unchanged. + */ +export function withoutSqlStatementText(message: string, cause?: string | null): string { + let start = message.search(FAILED_QUERY_MARKER); + if (start < 0 && SQL_PLACEHOLDER.test(message)) start = SQL_STATEMENT.exec(message)?.index ?? -1; + if (start < 0) return message; + const prefix = message.slice(0, start).replace(/[\s:;,-]+$/, "").trim(); + const safeCause = normalizeRunFailureCause(cause); + const replacement = `Database query failed${safeCause ? ` (${safeCause})` : ""}`; + return prefix ? `${prefix}: ${replacement}` : replacement; +} diff --git a/server/src/__tests__/activity-service.test.ts b/server/src/__tests__/activity-service.test.ts index 713054457bf..54c4d26bd62 100644 --- a/server/src/__tests__/activity-service.test.ts +++ b/server/src/__tests__/activity-service.test.ts @@ -365,6 +365,28 @@ describeEmbeddedPostgres("activity service", () => { }, ); + it.each([ + ["recorded", "40001", "40001"], + ["legacy row without the field", undefined, null], + ["malformed", 'Failed query: select "secret" from x', null], + ["overlong", "X".repeat(81), null], + ["boolean", true, null], + ["number", 40001, null], + ] as const)("projects only a closed-shape failure cause in issue history: %s", async (_case, stored, expected) => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Failure cause", issuePrefix: `F${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Failing runner", adapterType: "process" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Fail with a cause", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, scopeKind: "issue", status: "failed", + errorCode: "adapter_failed", finishedAt: new Date(), contextSnapshot: { issueId }, livenessState: "failed", + resultJson: { stopReason: "adapter_failed", ...(stored === undefined ? {} : { failureCause: stored }) } }); + + const [row] = await activityService(db).runsForIssue(companyId, issueId); + expect(row).toMatchObject({ runId, errorCode: "adapter_failed", failureCause: expected }); + expect(row!.resultJson).toEqual({ stopReason: "adapter_failed" }); + expect(JSON.stringify(row)).not.toContain("secret"); + }); + it("backfills missing liveness for completed issue runs before returning the ledger", async () => { const companyId = randomUUID(); const agentId = randomUUID(); diff --git a/server/src/__tests__/agent-secrets-routes.test.ts b/server/src/__tests__/agent-secrets-routes.test.ts index 4dbc5b52ab2..6e0e94dfeb6 100644 --- a/server/src/__tests__/agent-secrets-routes.test.ts +++ b/server/src/__tests__/agent-secrets-routes.test.ts @@ -267,6 +267,19 @@ describeEmbeddedPostgres("agent secret routes", () => { .toBe(redacted[0].text); }); + it("drops SQL statement text from run errors on every run read", async () => { + const fixture = await seedAgentRun(); + const registry = createRunSecretRedactionRegistry(db); + const legacy = { id: fixture.heartbeatRunId, status: "failed", errorCode: "adapter_failed", + error: 'Failed query: select "token" from "secrets" where "id" = $1\nparams: sk-secret', resultJson: { failureCause: "40001" } }; + const [listed] = await registry.redactForRuns(fixture.companyId, [legacy]); + expect(listed).toMatchObject({ id: fixture.heartbeatRunId, error: "Database query failed (40001)", resultJson: { failureCause: "40001" } }); + expect(await registry.redactForRun(fixture.companyId, fixture.heartbeatRunId, legacy)) + .toMatchObject({ error: "Database query failed (40001)" }); + const plain = { id: fixture.heartbeatRunId, error: "Adapter failed" }; + expect((await registry.redactForRuns(fixture.companyId, [plain]))[0]).toEqual(plain); + }); + it("denies low-trust, task-bridge, and skill-test callers on both routes", async () => { const lowTrust = await seedAgentRun({ trustPreset: LOW_TRUST_REVIEW_PRESET, diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index 2ab01f77b2d..4c4508e610e 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -1091,6 +1091,30 @@ describeEmbeddedPostgres("attention service", () => { expect(JSON.stringify(plans.generic)).toContain("heartbeat_run_events_retry_exhausted_idx"); }); + it("keeps SQL statement text saved by older failed runs out of the attention inbox", async () => { + const { companyId, workerId } = await seedCompany("ATQ"); + const issueId = await insertIssue({ companyId, identifier: "ATQ-1", title: "Failed with a query", status: "in_progress" }); + const failedRunId = randomUUID(); + const failedAt = new Date("2026-07-09T12:00:00.000Z"); + await db.insert(heartbeatRuns).values({ + id: failedRunId, companyId, agentId: workerId, invocationSource: "automation", status: "failed", + errorCode: "adapter_failed", + error: 'Usage write failed: Failed query: update "heartbeat_runs" set "usage_json" = $1\nparams: {"apiKey":"sk-secret"}', + contextSnapshot: { issueId }, createdAt: failedAt, updatedAt: failedAt, finishedAt: failedAt, + }); + await db.insert(heartbeatRunEvents).values({ + companyId, runId: failedRunId, agentId: workerId, seq: 1, eventType: "lifecycle", + message: "Bounded retry exhausted after 4 scheduled attempts; no further automatic retry will be queued", + createdAt: new Date("2026-07-09T12:00:01.000Z"), + }); + + const feed = await attentionService(db).list(companyId, { userId: "board-user" }); + const item = feed.items.find((entry) => entry.sourceKind === "failed_run" && entry.subject.id === failedRunId); + expect(item?.detail).toMatchObject({ failureReasonExcerpt: "Usage write failed: Database query failed" }); + expect(item?.subject.metadata?.error).toBe("Usage write failed: Database query failed"); + expect(JSON.stringify(item)).not.toMatch(/heartbeat_runs|sk-secret|params/); + }); + it("suppresses failed-run attention after a newer run for the same issue", async () => { const { companyId, workerId } = await seedCompany("ATN"); const issueId = await insertIssue({ diff --git a/server/src/__tests__/issue-continuation-summary.test.ts b/server/src/__tests__/issue-continuation-summary.test.ts index 73cf9ba4e49..5e4815c8c43 100644 --- a/server/src/__tests__/issue-continuation-summary.test.ts +++ b/server/src/__tests__/issue-continuation-summary.test.ts @@ -86,6 +86,19 @@ describe("issue continuation summaries", () => { expect(body).toContain("Inspect the failed run, fix the cause"); }); + it("keeps SQL statement text saved by older runs out of the summary", () => { + const body = buildContinuationSummaryMarkdown({ + issue: { id: "issue-1", identifier: "PAP-1579", title: "Add continuation summaries", description: null, + status: "in_progress", priority: "medium" }, + run: { id: "run-3", status: "failed", errorCode: "adapter_failed", resultJson: { failureCause: "40001" }, + error: 'Usage write failed: Failed query: update "heartbeat_runs" set "usage_json" = $1\nparams: {"apiKey":"sk-secret"}' }, + agent: { id: "agent-1", name: "CodexCoder", adapterType: "codex_local" }, + }); + + expect(body).toContain("Latest run error (adapter_failed): Usage write failed: Database query failed (40001)"); + expect(body).not.toMatch(/heartbeat_runs|sk-secret/); + }); + it("detects continuation summaries that explicitly park executor work for review", () => { const body = [ "# Continuation Summary", diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts index c4975395ed8..3b8a5f1f105 100644 --- a/server/src/services/activity.ts +++ b/server/src/services/activity.ts @@ -20,7 +20,7 @@ import { issueWorkProducts, workspaceOperations, } from "@paperclipai/db"; -import { hasWorkspaceRestoreFailure, safeWorkspaceRestorePath, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared"; +import { hasWorkspaceRestoreFailure, safeWorkspaceRestorePath, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, normalizeRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH } from "@paperclipai/shared"; import { logger } from "../middleware/logger.js"; import { visibleIssueCondition } from "./issue-visibility.js"; import { classifyRunLiveness } from "./run-liveness.js"; @@ -477,6 +477,9 @@ export function activityService(db: Db) { then 'This credential is not shared with the responsible user' else null end`, usageJson: summarizedUsageJson, resultJson: summarizedResultJson(viewerUserId), + // Bounded cause of a generic failure code; validated below, legacy rows have none. + failureCause: sql`case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'failureCause') = 'string' + then left(${heartbeatRuns.resultJson} ->> 'failureCause', ${RUN_FAILURE_CAUSE_MAX_LENGTH + 1}) end`, logBytes: heartbeatRuns.logBytes, retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, @@ -603,6 +606,7 @@ export function activityService(db: Db) { : null; return { ...run, + failureCause: normalizeRunFailureCause(run.failureCause), resultJson: run.resultJson ? { ...run.resultJson, ...(Object.hasOwn(run.resultJson, "workspaceRestorePath") ? { diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts index 44f2c844cf1..db8206e9376 100644 --- a/server/src/services/attention.ts +++ b/server/src/services/attention.ts @@ -51,6 +51,7 @@ import type { IssueThreadInteractionResolverPolicyProvenance, IssueReviewPolicy, } from "@paperclipai/shared"; +import { withoutSqlStatementText } from "@paperclipai/shared"; import { badRequest } from "../errors.js"; import { listAttentionExhaustedRuns } from "./attention-exhausted-runs.js"; import { budgetService } from "./budgets.js"; @@ -1731,7 +1732,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions agentName: run.agentName, issueId, errorCode: run.errorCode, - error: run.error, + error: run.error ? withoutSqlStatementText(run.error) : run.error, retryExhaustedReason: run.exhaustionMessage, }, }, @@ -1754,7 +1755,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions detail: { kind: "failed_run", agentName: run.agentName, - failureReasonExcerpt: excerpt(run.error ?? run.exhaustionMessage ?? run.errorCode), + // Rows saved before SQL text was kept out of run errors still carry it. + failureReasonExcerpt: excerpt((run.error ? withoutSqlStatementText(run.error) : null) ?? run.exhaustionMessage ?? run.errorCode), images: issueImages(failedImageMap, issueId), }, })); diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index f0b8fbe5dce..c769f295123 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -94,7 +94,8 @@ async function hasSettledFailedPreparationEvidence(db: Db, run: Run, historicalR || run.resultJson?.stopReason !== "adapter_failed" || Object.keys(run.resultJson).some(key => !["stopReason", "timeoutFired", "timeoutSource", "executionRecovery", "timeoutConfigured", "effectiveTimeoutSec", "systemRecoveryAdoption", "workspaceRestoreFailure", - "workspaceRestoreRecovery"].includes(key))) return false; + // A bounded diagnostic code; it carries no execution or replay authority. + "workspaceRestoreRecovery", "failureCause"].includes(key))) return false; return Boolean(await readFailedPreparationSettlement(db, { ...run, agentId: source.agentId }, historicalRetryAncestor)); } diff --git a/server/src/services/chat-channels/failed-run-retry.ts b/server/src/services/chat-channels/failed-run-retry.ts index 573425394ab..38e62ea7157 100644 --- a/server/src/services/chat-channels/failed-run-retry.ts +++ b/server/src/services/chat-channels/failed-run-retry.ts @@ -244,7 +244,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti Object.keys(recovery).every(key => ["kind", "providerWorkStarted"].includes(key)) && run.resultJson?.stopReason === "adapter_failed" && Object.keys(run.resultJson).every(key => ["executionRecovery", ...(workspaceValidation ? ["workspaceValidation"] : []), "stopReason", - "effectiveTimeoutSec", "timeoutConfigured", "timeoutFired", "timeoutSource"].includes(key)); + "effectiveTimeoutSec", "timeoutConfigured", "timeoutFired", "timeoutSource", "failureCause"].includes(key)); const diagnostic = failedRetry ? retryDiagnostic : workspacePreparation ? run.error : "reviewed_chat_execution_binding_not_authorized"; const dispatchOnlyProfile = failedRetry || workspacePreparation; if ( diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index ee6fdd737b7..3352d60d9b8 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -9,7 +9,7 @@ import { agentWakeupRequests, chatConversations, chatEndpoints, environmentLease import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js"; import { canRetryStoppedRun } from "./cancelled-native-startup.js"; import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; -import { EXECUTION_RECONCILIATION_CAUSES, type ExecutionBlocker } from "@paperclipai/shared"; +import { EXECUTION_RECONCILIATION_CAUSES, readRunFailureCause, withoutSqlStatementText, type ExecutionBlocker } from "@paperclipai/shared"; import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA, nativeRunHasNoWorkspaceAccess } from "./workspace-restore-recovery-state.js"; import { unreconciledNativeExportPredicate } from "./native-runtime/native-workspace-reconstruction.js"; import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; @@ -388,7 +388,8 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st ? "Waiting for review; this continuation never started." : cancellation?.reason ?? (run?.status === "cancelled" ? "Execution was cancelled; its source was not recorded." - : run?.error); + // Rows written before SQL text was kept out of run errors still carry it. + : run?.error ? withoutSqlStatementText(run.error, readRunFailureCause(run.resultJson)) : run?.error); const [chatBinding] = eligibleContinuation || canRetry ? await db.select({ state: chatConversations.state, endpointStatus: chatEndpoints.status, connectionStatus: toolConnections.status, connectionEnabled: toolConnections.enabled, diff --git a/server/src/services/fresh-workspace-continuation.ts b/server/src/services/fresh-workspace-continuation.ts index e5ae5132b6a..c912df92970 100644 --- a/server/src/services/fresh-workspace-continuation.ts +++ b/server/src/services/fresh-workspace-continuation.ts @@ -223,7 +223,7 @@ async function reusableDirectoryAfterPreparationFailure(db: Db, source: typeof h source.lastOutputSeq !== 0 || source.logCompressed || Object.keys(source.usageJson ?? {}).some(key => key !== "ledgerScope") || Object.keys(source.runnerProfileJson ?? {}).some(key => !["adapterDispatch", "chatControlRecoveryAdmission", "aiConnectionNonAssigneeCommentWake"].includes(key)) || - Object.keys(source.resultJson ?? {}).some(key => !["effectiveTimeoutSec", "executionRecovery", "stopReason", "timeoutConfigured", "timeoutFired", "timeoutSource", "workspaceValidation"].includes(key))) return null; + Object.keys(source.resultJson ?? {}).some(key => !["effectiveTimeoutSec", "executionRecovery", "stopReason", "timeoutConfigured", "timeoutFired", "timeoutSource", "workspaceValidation", "failureCause"].includes(key))) return null; const settlement = await readFailedPreparationSettlement(db, source); const warning = settlement?.unsupportedHarnessWarning; if (!settlement || warning && (warning.message !== "Company observability was not applied: this harness is not supported." diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index cd9df2ae118..7ce45027e9f 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -823,6 +823,56 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( error: STORAGE_UNAVAILABLE_EXHAUSTED_MESSAGE }); }); + it.each(["execution", "setup"] as const)("names the nested cause of a generic %s failure and keeps SQL text off the board", async phase => { + const f = await fixture(); + const logged = vi.spyOn(logger, "error"); + onTestFinished(() => logged.mockRestore()); + const driver = Object.assign(new Error("could not serialize access due to concurrent update"), { code: "40001" }); + const query = new Error('Failed query: update "heartbeat_runs" set "usage_json" = $1 where "id" = $2\nparams: {"apiKey":"sk-live-secret"},run-1', + { cause: new Error("transaction aborted", { cause: driver }) }); + const error = new Error(`Usage write failed: ${query.message}`, { cause: query }); + if (phase === "setup") await f.completion.failRunSetup(f.setup(error)); + else await f.completion.failRun(f.failure(error)); + const run = await read(f.run.id); + expect(run).toMatchObject({ status: "failed", errorCode: phase === "setup" ? "setup_failed" : "adapter_failed", + error: "Usage write failed: Database query failed (40001)", resultJson: { failureCause: "40001" } }); + expect(JSON.stringify(run.resultJson)).not.toMatch(/heartbeat_runs|sk-live-secret|params/); + expect(logged).toHaveBeenCalledWith(expect.objectContaining({ runId: f.run.id, failureCause: "40001" }), expect.any(String)); + }); + + it("bounds the failure cause to a closed-shape token and falls back to the error class", async () => { + const f = await fixture(); + class QueryTimeoutError extends Error { override name = "QueryTimeoutError"; } + const unsafe = Object.assign(new QueryTimeoutError("timed out"), { code: `${"X".repeat(81)}` }, + { cause: { code: "select * from users" } }); + await f.completion.failRun(f.failure(unsafe)); + expect((await read(f.run.id)).resultJson).toMatchObject({ failureCause: "QueryTimeoutError" }); + }); + + it("names why an adapter result failed without a code", async () => { + const f = await fixture(); + await f.completion.completeRun({ ...f.input, adapterResult: { ...f.input.adapterResult, exitCode: 137, signal: null, errorMessage: null } }); + expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "adapter_failed", error: "Adapter failed", + resultJson: { stopReason: "adapter_failed", failureCause: "exit_code_137" } }); + }); + + it("keeps SQL text out of a cancelled adapter result's error", async () => { + const f = await fixture(); + await f.completion.completeRun({ ...f.input, adapterResult: { ...f.input.adapterResult, exitCode: 1, + errorMessage: 'Failed query: select "secret_value" from x where id = $1\nparams: sk-secret', resultJson: { status: "cancelled" } } }); + const run = await read(f.run.id); + expect(run.status).toBe("cancelled"); + expect(run.error).toBe("Database query failed"); + }); + + it("records no failure cause when the failure already has a named code", async () => { + const f = await fixture(); + await f.completion.completeRun({ ...f.input, adapterResult: { ...f.input.adapterResult, exitCode: 1, errorCode: "provider_quota", errorMessage: "quota" } }); + const run = await read(f.run.id); + expect(run.errorCode).toBe("provider_quota"); + expect(run.resultJson).not.toHaveProperty("failureCause"); + }); + it("keeps an ordinary adapter failure on the normal recovery path", async () => { const f = await fixture(); await f.completion.failRun(f.failure(new Error("adapter crashed"))); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 072df99604f..9e536677957 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1,6 +1,8 @@ import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; import { isContextCompactionRun } from "../context-compaction.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; +import { GENERIC_RUN_FAILURE_CODES, failureCauseFromAdapterResult, failureCauseFromError } from "./run-failure-cause.js"; +import { withoutSqlStatementText } from "@paperclipai/shared"; import { nativeTaskSessionPublicationHandled } from "../native-runtime/native-task-session-publication.js"; import { boundOriginalMessage, describeRefusedExactResume, interruptedExactResumeCause } from "../native-runtime/native-exact-resume-failure.js"; import { settledResponsePublicationSql } from "../native-runtime/native-response-publication-receipt.js"; @@ -477,12 +479,12 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const normalizedUsage = sessionUsageResolution.normalizedUsage; const runErrorMessage = outcome === "cancelled" - ? redactCurrentUserText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled", currentUserRedactionOptions) + ? redactCurrentUserText(withoutSqlStatementText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled"), currentUserRedactionOptions) : outcome === "succeeded" ? null : redactCurrentUserText( - adapterResult.errorMessage ?? - (outcome === "timed_out" ? "Timed out" : "Adapter failed"), + withoutSqlStatementText(adapterResult.errorMessage ?? + (outcome === "timed_out" ? "Timed out" : "Adapter failed")), currentUserRedactionOptions, ); const recordedResponsibleUserDenialCode = @@ -497,6 +499,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC recordedResponsibleUserDenialCode ?? "adapter_failed") : null; + // A generic code alone does not say what failed; record a bounded cause. + const failureCause = outcome === "failed" && runErrorCode && GENERIC_RUN_FAILURE_CODES.has(runErrorCode) + ? failureCauseFromAdapterResult(adapterResult, { processCancellationFailed: Boolean(processCancellation?.failed) }) + : null; + if (failureCause) { + logger.warn({ runId: run.id, errorCode: runErrorCode, failureCause }, "heartbeat run failed"); + } async function persistSessionState(finalizedRun: Run) { await updateRuntimeState( @@ -650,6 +659,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ? { executionRecovery: adapterResult.executionRecovery } : {}), ...(outcome === "cancelled" ? controllerLeaseLossResultJson(signal) : {}), + ...(failureCause ? { failureCause } : {}), configFreshness: configFreshnessResultMetadata, }, errorFamily: adapterResult.errorFamily ?? null, @@ -1097,7 +1107,10 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const redactFailureText = (text: string) => redactCurrentUserText(identityRedactor.redact(text), currentUserRedactionOptions); - const thrownMessage = redactFailureText(err instanceof Error ? err.message : "Unknown adapter failure"); + // Bounded code from the error or its first three causes; never message text. + const thrownFailureCause = failureCauseFromError(err, providerFailure); + const thrownMessage = withoutSqlStatementText( + redactFailureText(err instanceof Error ? err.message : "Unknown adapter failure"), thrownFailureCause); const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err : null; @@ -1155,7 +1168,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC nativeTerminalFailureCode ?? (isStorageUnavailableError(err) ? STORAGE_UNAVAILABLE_CODE : null) ?? "adapter_failed"; - logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId }, "heartbeat execution failed"); + const failureCause = GENERIC_RUN_FAILURE_CODES.has(failureErrorCode) ? thrownFailureCause : null; + logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId, + errorCode: failureErrorCode, failureCause: thrownFailureCause }, "heartbeat execution failed"); let logSummary: { bytes: number | null; @@ -1195,6 +1210,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC errorMessage: thrownMessage, resultJson: { ...parseObject(stopSnapshot?.resultJson), + ...(failureCause && !stoppedDuringFailure ? { failureCause } : {}), ...(providerFailure ? { providerFailure: { ...providerFailure, message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...(interruptedExactResume ? { nativeExactResumeRefusal: { @@ -1381,12 +1397,14 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const providerFailure = providerFailureFromError(outerErr); const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const storageUnavailable = isStorageUnavailableError(outerErr); - const message = storageUnavailable ? storageUnavailableMessage(outerErr) : redactCurrentUserText( + // Bounded code from the error or its first three causes; never message text. + const thrownFailureCause = failureCauseFromError(outerErr, providerFailure); + const message = storageUnavailable ? storageUnavailableMessage(outerErr) : withoutSqlStatementText(redactCurrentUserText( identityRedactor.redact(outerErr instanceof Error ? outerErr.message : "Unknown setup failure"), currentUserRedactionOptions, - ); + ), thrownFailureCause); // A missing secret/env binding is a known pre-dispatch configuration gap, // not an opaque setup crash. Surface it with its own errorCode so the // recovery path routes it to a human owner instead of looping retries. @@ -1469,6 +1487,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC recordedResponsibleUserDenialCode ?? nonRetryablePreflightCode ?? "setup_failed"; + const failureCause = GENERIC_RUN_FAILURE_CODES.has(setupFailureErrorCode) ? thrownFailureCause : null; // Bounded kind and reason only: a workspace provider's own text never crosses into Core. const workspaceProviderFailure = findWorkspaceProviderFailure(outerErr)?.summary() ?? null; // A wrapped query error hides its driver cause (for example @@ -1477,7 +1496,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const setupFailureCauseCode = typeof causeCode === "string" && /^[A-Z0-9_]{1,64}$/.test(causeCode) ? causeCode : null; logger.error( { err: identityRedactor.redact({ message: outerErr instanceof Error ? outerErr.message : String(outerErr), stack: outerErr instanceof Error ? outerErr.stack : undefined }), runId, - errorCode: setupFailureErrorCode, ...(setupFailureCauseCode ? { causeCode: setupFailureCauseCode } : {}), + errorCode: setupFailureErrorCode, failureCause: thrownFailureCause, ...(setupFailureCauseCode ? { causeCode: setupFailureCauseCode } : {}), ...(workspaceProviderFailure ? { workspaceProviderFailure } : {}) }, "heartbeat execution setup failed", ); @@ -1512,6 +1531,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const connectionFailure = readGitConnectionFailure(outerErr); const setupFailureResultJson = { ...setupFailureDetails, + ...(failureCause ? { failureCause } : {}), ...(repositoryCatalogUnavailable ? { errorFamily: "transient_upstream", ...(repositoryCatalogUnavailable.retryAt ? { retryNotBefore: new Date(repositoryCatalogUnavailable.retryAt).toISOString() } : {}), } : {}), diff --git a/server/src/services/heartbeat/run-failure-cause.test.ts b/server/src/services/heartbeat/run-failure-cause.test.ts new file mode 100644 index 00000000000..6b5ff341efe --- /dev/null +++ b/server/src/services/heartbeat/run-failure-cause.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { normalizeRunFailureCause } from "@paperclipai/shared"; +import { failureCauseFromAdapterResult, failureCauseFromError } from "./run-failure-cause.js"; + +describe("run failure cause", () => { + it("reads the first safe code from the error and at most three nested causes", () => { + const wrap = (cause: unknown, depth: number): unknown => depth === 0 ? cause : new Error(`wrapper ${depth}`, { cause: wrap(cause, depth - 1) }); + expect(failureCauseFromError(wrap({ code: "ECONNRESET" }, 3))).toBe("ECONNRESET"); + // A fourth nested cause is outside the bound: the outer class name is used. + expect(failureCauseFromError(wrap({ code: "ECONNRESET" }, 4))).toBe("Error"); + expect(failureCauseFromError(Object.assign(new Error("x"), { code: "40001" }))).toBe("40001"); + }); + + it("prefers the provider's HTTP status, then its category", () => { + expect(failureCauseFromError(new Error("x"), { httpStatus: 429, category: "rate_limit" })).toBe("provider_http_429"); + expect(failureCauseFromError(new Error("x"), { category: "authentication" })).toBe("provider_authentication"); + expect(failureCauseFromError(new TypeError("x"), { category: "unknown" })).toBe("TypeError"); + }); + + it("never returns message text and falls back for non-errors", () => { + expect(failureCauseFromError({ code: "relation \"users\" does not exist" })).toBe("object"); + expect(failureCauseFromError("Failed query: select 1")).toBe("string"); + expect(failureCauseFromError(Object.assign(new Error("x"), { name: "bad name with spaces" }))).toBe("unknown_error"); + for (const value of [null, undefined, 42, new RangeError("y")]) { + expect(normalizeRunFailureCause(failureCauseFromError(value))).not.toBeNull(); + } + }); + + it("names adapter results by family, signal, cancellation, exit code, or reported error", () => { + const base = { errorFamily: null, signal: null, exitCode: 0, errorMessage: null }; + expect(failureCauseFromAdapterResult({ ...base, errorFamily: "transient_upstream", exitCode: 1 })).toBe("transient_upstream"); + expect(failureCauseFromAdapterResult({ ...base, signal: "SIGKILL", exitCode: 137 })).toBe("signal_SIGKILL"); + expect(failureCauseFromAdapterResult(base, { processCancellationFailed: true })).toBe("process_cancellation_failed"); + expect(failureCauseFromAdapterResult({ ...base, exitCode: 2 })).toBe("exit_code_2"); + expect(failureCauseFromAdapterResult({ ...base, errorMessage: "select * from secrets where id = $1" })).toBe("adapter_reported_error"); + expect(failureCauseFromAdapterResult(base)).toBe("unknown_error"); + }); +}); diff --git a/server/src/services/heartbeat/run-failure-cause.ts b/server/src/services/heartbeat/run-failure-cause.ts new file mode 100644 index 00000000000..257f5763560 --- /dev/null +++ b/server/src/services/heartbeat/run-failure-cause.ts @@ -0,0 +1,48 @@ +import { normalizeRunFailureCause } from "@paperclipai/shared"; +import type { AdapterExecutionResult } from "@paperclipai/adapter-utils"; +import { safeErrorCauseCode } from "../../vendor/paperclip-runner/index.js"; + +/** + * Bounded diagnosis for runs that settle with a generic failure code + * (`adapter_failed`, `setup_failed`). The code stays generic because callers + * compare against it; `resultJson.failureCause` names what went wrong with a + * closed-shape token (see `@paperclipai/shared` run-failure-cause). It is + * derived only from error codes, class names, provider HTTP status, process + * exit facts and enum families, never from message text. + */ +export const GENERIC_RUN_FAILURE_CODES: ReadonlySet = new Set(["adapter_failed", "setup_failed"]); + +export const UNKNOWN_RUN_FAILURE_CAUSE = "unknown_error"; + +type ProviderFailureFacts = { httpStatus?: number; category?: string } | null | undefined; + +/** A thrown failure: the provider's HTTP status or category, else the error's or its first three causes' code, else its class name. */ +export function failureCauseFromError(error: unknown, providerFailure?: ProviderFailureFacts): string { + const httpStatus = providerFailure?.httpStatus; + if (typeof httpStatus === "number" && Number.isInteger(httpStatus) && httpStatus >= 400 && httpStatus <= 599) { + return `provider_http_${httpStatus}`; + } + const category = providerFailure?.category; + if (category && category !== "unknown") { + const cause = normalizeRunFailureCause(`provider_${category}`); + if (cause) return cause; + } + return normalizeRunFailureCause(safeErrorCauseCode(error)) ?? UNKNOWN_RUN_FAILURE_CAUSE; +} + +/** An adapter that returned a failed result without a named code: its error family, signal or exit code. */ +export function failureCauseFromAdapterResult( + result: Pick, + options: { processCancellationFailed?: boolean } = {}, +): string { + const family = normalizeRunFailureCause(result.errorFamily); + if (family) return family; + const signal = result.signal ? normalizeRunFailureCause(`signal_${result.signal}`) : null; + if (signal) return signal; + if (options.processCancellationFailed) return "process_cancellation_failed"; + if (typeof result.exitCode === "number" && Number.isInteger(result.exitCode) && result.exitCode !== 0) { + return `exit_code_${result.exitCode}`; + } + if (result.errorMessage) return "adapter_reported_error"; + return UNKNOWN_RUN_FAILURE_CAUSE; +} diff --git a/server/src/services/issue-continuation-summary.ts b/server/src/services/issue-continuation-summary.ts index 554c8313bab..986aedb5bf3 100644 --- a/server/src/services/issue-continuation-summary.ts +++ b/server/src/services/issue-continuation-summary.ts @@ -1,7 +1,7 @@ import { and, eq } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { documents, issueDocuments, issues } from "@paperclipai/db"; -import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, type SourceTrustMetadata } from "@paperclipai/shared"; +import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, readRunFailureCause, withoutSqlStatementText, type SourceTrustMetadata } from "@paperclipai/shared"; import { documentService } from "./documents.js"; import { summarizeRunErrorForModel } from "./heartbeat-run-summary.js"; @@ -146,7 +146,9 @@ export function buildContinuationSummaryMarkdown(input: { const terminalFailureCategory = terminalFailure && typeof terminalFailure === "object" && !Array.isArray(terminalFailure) ? ((terminalFailure as Record).category ?? "unknown") : null; - const modelError = summarizeRunErrorForModel(run.error, terminalFailureCategory); + // Rows saved before SQL text was kept out of run errors still carry it. + const modelError = summarizeRunErrorForModel( + run.error ? withoutSqlStatementText(run.error, readRunFailureCause(run.resultJson)) : run.error, terminalFailureCategory); const recentActions = [ `Run \`${run.id}\` finished with status \`${run.status}\`${run.finishedAt ? ` at ${run.finishedAt.toISOString()}` : ""}.`, resultSummary ? truncateText(resultSummary, SUMMARY_SECTION_MAX_CHARS) : "No adapter-provided result summary was captured for this run.", diff --git a/server/src/services/run-secret-redaction.ts b/server/src/services/run-secret-redaction.ts index 306b529f39b..725d7342382 100644 --- a/server/src/services/run-secret-redaction.ts +++ b/server/src/services/run-secret-redaction.ts @@ -6,6 +6,20 @@ import { REDACTED_EVENT_VALUE } from "../redaction.js"; import { getSecretProvider } from "../secrets/provider-registry.js"; import type { StoredSecretVersionMaterial } from "../secrets/types.js"; import { credentialAccessConnectionNameSql } from "./credential-access-visibility.js"; +import { readRunFailureCause, withoutSqlStatementText } from "@paperclipai/shared"; + +/** + * A run row's `error` without SQL statement text. Rows saved before run errors + * were cleaned at settlement can still carry a Drizzle `Failed query:` statement + * and its parameters; every run read that passes through here drops them. + */ +function withoutSqlRunError(value: T): T { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const record = value as Record; + if (typeof record.error !== "string") return value; + const error = withoutSqlStatementText(record.error, readRunFailureCause(record.resultJson)); + return error === record.error ? value : { ...record, error } as T; +} const REGISTRY_KEY = "paperclipSecretRedactions"; // Project only the registry: run contexts can contain megabytes of prompt data. @@ -145,12 +159,12 @@ export function createRunSecretRedactionRegistry(db: Db) { }))); return runs.map((run) => { const disclosure = valuesByRun.get(run.id); - return redactRegisteredSecretValues(run, disclosure?.values ?? [], disclosure?.credentialConnectionName ?? null); + return redactRegisteredSecretValues(withoutSqlRunError(run), disclosure?.values ?? [], disclosure?.credentialConnectionName ?? null); }); }, redactForRun: async (companyId: string, runId: string, value: T, viewerUserId: string | null = null): Promise => { const disclosure = await valuesForRun(companyId, runId, viewerUserId); - return redactRegisteredSecretValues(value, disclosure.values, disclosure.credentialConnectionName); + return redactRegisteredSecretValues(withoutSqlRunError(value), disclosure.values, disclosure.credentialConnectionName); }, redactForIssue: async (companyId: string, issueId: string, value: T): Promise => redactRegisteredSecretValues(value, await valuesForIssue(companyId, issueId)), diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 6809d30e609..1f75c4d8292 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -104,7 +104,7 @@ async function readBoardMessageRetrySource(db: Db, input: { sql`${heartbeatRuns.resultJson}->>'stopReason' = 'adapter_failed'`, sql`not exists (select 1 from jsonb_object_keys(case when jsonb_typeof(${heartbeatRuns.resultJson}) = 'object' then ${heartbeatRuns.resultJson} else '{}'::jsonb end) key - where key not in ('effectiveTimeoutSec', 'executionRecovery', 'stopReason', 'timeoutConfigured', 'timeoutFired', 'timeoutSource'))`, + where key not in ('effectiveTimeoutSec', 'executionRecovery', 'stopReason', 'timeoutConfigured', 'timeoutFired', 'timeoutSource', 'failureCause'))`, sql`${heartbeatRuns.runnerProfileJson}->'adapterDispatch' = '{"adapterType":"paperclip_runner"}'::jsonb`, sql`not exists (select 1 from jsonb_object_keys(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) key diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 76be2f77175..197732fcf32 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -93,6 +93,7 @@ export const CURSOR_CLOUD_DEFAULT_TIMEOUT_MS = runner.CURSOR_CLOUD_DEFAULT_TIMEO export const RUNNER_TOOL_SCOPE_TOKEN_HEADER = runner.RUNNER_TOOL_SCOPE_TOKEN_HEADER; export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const providerFailureFromError = runner.providerFailureFromError; +export const safeErrorCauseCode = runner.safeErrorCauseCode; export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint; export const captureLegacyDurablePrpCheckpoint = runner.captureLegacyDurablePrpCheckpoint; export const inspectDurablePrpCheckpointHeader = runner.inspectDurablePrpCheckpointHeader; diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 7e9cb211a11..5451c5cf1bc 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -644,7 +644,7 @@ its workload. They catch a stuck or misleading task that no scenario anticipated | --- | --- | --- | | U1 liveness | a non-terminal task has neither a live owner (a running run with recent evidence, a run queued within the queue bound, a scheduled retry, an accepted Retry that waits on the system within the queue bound, a system recovery action) nor a visible, actionable item (Retry, a question or `userAction`, a failed notice, a board recovery action, an execution blocker, `blocked` or `in_review`; none of these count while an accepted Retry waits on the system) for longer than the bound, counting only time with the server ready | 3 min ownerless (`RESTART_CHAOS_U1_ORPHAN_SECONDS`); a queued run or an accepted Retry's wait counts as an owner for 5 min (`RESTART_CHAOS_U1_QUEUED_SECONDS`) | | U2 retry loop | more than N consecutive failed runs with the same status and code while the issue status stays the same | 3 (`RESTART_CHAOS_U2_SAME_FAILURE_RUNS`) | -| U3 diagnosable | a failed run has no code or a generic one (`adapter_failed`, `unknown`, `error`, ...), its stored error is empty or names only a bare cause (`last cause: Error`), or a recovery action's cause or next action is empty or bare | none | +| U3 diagnosable | a failed run has no code or a generic one (`adapter_failed`, `unknown`, `error`, ...) and no recorded `failureCause` (a bounded code such as `40001`, `exit_code_137` or `provider_http_429`; a bare `Error` or `unknown_error` does not count), its stored error is empty or names only a bare cause (`last cause: Error`), or a recovery action's cause or next action is empty or bare | none | | U4 stale notice | a Retry or failed notice stays on a task after its latest run succeeded, longer than the bound or at rest | 60 s (`RESTART_CHAOS_U4_STALE_SECONDS`) | U1, U2 and U4 fail the task (signature `U:...`). U3 is a warning: it is reported in each grade @@ -675,7 +675,9 @@ only until its run is dispatched. It also flags ten tasks whose accepted Retry w for 8 minutes without starting a run (five in saved waves, five in the L1 waves above). U2 flags a claude task that failed four times in a row with `adapter_failed`. Every U3 flag is `adapter_failed`, including three tasks that passed after a retry, which is why U3 stays a -warning until `adapter_failed` names its cause. The audit saw only failure codes; the error-text +warning until `adapter_failed` names its cause. Runs now record that cause as `failureCause` (shown +by the board's run list), and U3 accepts it; saved waves from before the field show none, so U3 +remains a warning until fresh waves show it holds. The audit saw only failure codes; the error-text part of U3 is checked only in waves run with this harness. ## Time faults (TM1–TM3, `--time-age-hours`) diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index fc0f5bb3366..c2dcb4fc046 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { ChaosApi } from "./api.js"; -import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, STOP_CONTINUE_AFTER_MS, TaskObserver, childSpeedTasksInTurn, childrenPending, listWorkspaceFiles } from "./observer.js"; +import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, STOP_CONTINUE_AFTER_MS, TaskObserver, childSpeedTasksInTurn, childrenPending, listWorkspaceFiles, toRunSample } from "./observer.js"; import { gradeTask } from "./grader.js"; import { createFixtureRepository, type ChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; @@ -395,3 +395,11 @@ describe("childSpeedTasksInTurn", () => { }); }); +describe("run samples", () => { + it("keeps the board's failure cause for U3 and omits it for older servers", () => { + const base = { runId: "r1", status: "failed", errorCode: "adapter_failed" }; + expect(toRunSample({ ...base, failureCause: "40001" }).failureCause).toBe("40001"); + expect(toRunSample(base)).not.toHaveProperty("failureCause"); + expect(toRunSample({ ...base, failureCause: null })).not.toHaveProperty("failureCause"); + }); +}); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index c76446bf4ea..e1042cf6ed6 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -82,7 +82,9 @@ export function toRunSample(run: Json): RunSample { const wake = new Set([...list(run.wakeCommentIds), ...(typeof run.wakeCommentId === "string" ? [run.wakeCommentId] : [])]); return { id: String(run.runId ?? run.id), status: String(run.status), createdAt: run.createdAt ?? null, startedAt: run.startedAt ?? null, - finishedAt: run.finishedAt ?? null, errorCode: run.errorCode ?? null, retryOfRunId: run.retryOfRunId ?? null, + finishedAt: run.finishedAt ?? null, errorCode: run.errorCode ?? null, + ...(typeof run.failureCause === "string" && run.failureCause ? { failureCause: run.failureCause } : {}), + retryOfRunId: run.retryOfRunId ?? null, wakeCommentIds: [...wake], livenessState: run.livenessState ?? null, lastEvidenceAt: latestIso([run.startedAt, run.lastUsefulActionAt, run.execution?.lastConfirmedActivityAt]), }; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index d468301e086..dd985a3f678 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -190,6 +190,8 @@ export interface RunSample { startedAt: string | null; finishedAt: string | null; errorCode: string | null; + /** Bounded code naming why a generic `errorCode` (such as `adapter_failed`) failed, e.g. `40001`. */ + failureCause?: string | null; retryOfRunId: string | null; wakeCommentIds: string[]; livenessState: string | null; diff --git a/tests/restart-chaos/universal.test.ts b/tests/restart-chaos/universal.test.ts index c7a6eadbe90..87c84587ba3 100644 --- a/tests/restart-chaos/universal.test.ts +++ b/tests/restart-chaos/universal.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { gradeTask } from "./grader.js"; import type { OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; -import { bareCauseMessage, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, gradeU1, gradeU2, gradeU3, gradeU4, gradeUniversal, universalBounds } from "./universal.js"; +import { bareCauseMessage, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, diagnosableFailureCause, gradeU1, gradeU2, gradeU3, gradeU4, gradeUniversal, universalBounds } from "./universal.js"; import { workloadDefinitions } from "./workloads.js"; const T0 = Date.parse("2026-11-09T12:00:00Z"); @@ -128,6 +128,29 @@ describe("U3 diagnosable failures", () => { const coded = stalled.replace("last cause: Error", "last cause: 40001"); expect(gradeU3(record(replay), { runMessages: new Map([["r2", coded]]), recoveryActions: [] }).verdict).toBe("pass"); }); + it("treats a generic code with a recorded failure cause as diagnosable, and stays a warning", () => { + const failed = (failureCause?: string) => [sample(10 * SEC, "done", [run("r1", "failed", 0, 5 * SEC, + { errorCode: "adapter_failed", ...(failureCause === undefined ? {} : { failureCause }) })])]; + for (const cause of ["40001", "ECONNRESET", "provider_http_429", "exit_code_137", "TypeError"]) { + expect(gradeU3(record(failed(cause))).verdict).toBe("pass"); + } + // An old row without the field, or a cause that is itself bare, still names nothing. + for (const cause of [undefined, "", "Error", "unknown_error", "adapter_reported_error"]) { + const result = gradeU3(record(failed(cause))); + expect(result.verdict).toBe("fail"); + expect(result.detail).toContain("r1:code=adapter_failed"); + } + expect(gradeUniversal(record(failed("Error"))).U3).toMatchObject({ verdict: "fail", enforced: false }); + // The cause also excuses a bare stored message: "Adapter failed (40001)" names what went wrong. + expect(gradeU3(record(failed("40001")), { runMessages: new Map([["r1", "Error"]]), recoveryActions: [] }).verdict).toBe("pass"); + }); + it("recognizes diagnosable failure causes", () => { + expect(diagnosableFailureCause("40001")).toBe(true); + expect(diagnosableFailureCause("signal_SIGKILL")).toBe(true); + for (const cause of [null, undefined, "", " ", "Error", "unknown", "adapter_failed", "adapter_reported_error", "object"]) { + expect(diagnosableFailureCause(cause)).toBe(false); + } + }); it("flags a recovery action with an empty or bare cause", () => { const db = { runMessages: new Map(), recoveryActions: [{ id: "a1", kind: "stalled_issue", cause: "Error", nextAction: "Retry the task" }] }; expect(gradeU3(record([sample(0, "done", [])]), db).verdict).toBe("fail"); diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts index 83bf32c0af2..d48f44a78e9 100644 --- a/tests/restart-chaos/universal.ts +++ b/tests/restart-chaos/universal.ts @@ -5,7 +5,7 @@ * * U1 liveness every non-terminal task has a live owner or a visible, actionable item * U2 retry loop the same failure does not repeat without the task changing state - * U3 diagnosable every failed run names a specific cause + * U3 diagnosable every failed run names a specific cause (its code, or a recorded failureCause) * U4 stale notices no failure or Retry notice outlives later successful work * * Each rule is either `enforced` (a violation fails the task) or a warning @@ -78,6 +78,14 @@ const ready = (sample: TaskSample) => sample.reachable && sample.serverReady; * shows it as "adapter failed". */ export const GENERIC_FAILURE_CODES: ReadonlySet = new Set(["", "error", "unknown", "unknown_error", "failed", "adapter_failed", "internal_error"]); +/** Server fallbacks that record only that the adapter gave no code: they name nothing specific. */ +const VAGUE_FAILURE_CAUSES: ReadonlySet = new Set(["adapter_reported_error", "object", "string", "undefined"]); +/** A recorded failureCause that names something: not empty, a bare class, `unknown_error` or a no-code fallback. */ +export function diagnosableFailureCause(cause: string | null | undefined): boolean { + const text = (cause ?? "").trim(); + return text !== "" && !GENERIC_FAILURE_CODES.has(text.toLowerCase()) && !VAGUE_FAILURE_CAUSES.has(text) + && !bareCauseMessage(text); +} const FAILED_RUN = new Set(["failed", "timed_out"]); /** A message whose only cause is a bare exception class or "unknown" (the #20385 "last cause: Error" class). */ @@ -241,8 +249,11 @@ export function gradeU3(record: TaskRecord, db?: UniversalDbEvidence): Omit | null; resultJson: Record | null; logBytes?: number | null; diff --git a/ui/src/components/IssueRunLedger.test.tsx b/ui/src/components/IssueRunLedger.test.tsx index f94900ce410..71414b8a808 100644 --- a/ui/src/components/IssueRunLedger.test.tsx +++ b/ui/src/components/IssueRunLedger.test.tsx @@ -396,6 +396,39 @@ describe("IssueRunLedger", () => { expect(container.textContent).toContain("paused by board"); }); + it("names the recorded cause of an adapter failure and keeps the plain label for older runs", () => { + renderLedger({ + runs: [ + createRun({ + runId: "run-adapter-cause", + status: "failed", + errorCode: "adapter_failed", + failureCause: "40001", + resultJson: { stopReason: "adapter_failed" }, + }), + createRun({ + runId: "run-adapter-legacy", + status: "failed", + errorCode: "adapter_failed", + resultJson: { stopReason: "adapter_failed" }, + createdAt: "2026-04-18T19:57:00.000Z", + }), + createRun({ + runId: "run-adapter-malformed", + status: "failed", + errorCode: "adapter_failed", + failureCause: "select * from secrets", + resultJson: { stopReason: "adapter_failed" }, + createdAt: "2026-04-18T19:56:00.000Z", + }), + ], + }); + + expect(container.textContent).toContain("adapter failed (40001)"); + expect(container.textContent?.match(/adapter failed(?! \()/g)).toHaveLength(2); + expect(container.textContent).not.toContain("secrets"); + }); + it("surfaces active and completed child issue summaries", () => { renderLedger({ childIssues: [ diff --git a/ui/src/components/IssueRunLedger.tsx b/ui/src/components/IssueRunLedger.tsx index 76c94da2712..3ac205142aa 100644 --- a/ui/src/components/IssueRunLedger.tsx +++ b/ui/src/components/IssueRunLedger.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; import type { ActivityEvent, Issue, Agent, ProviderTraceMetadata } from "@paperclipai/shared"; import { isResponsibleUserDenialCode, + normalizeRunFailureCause, responsibleUserLabel, } from "@paperclipai/shared"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; @@ -329,7 +330,11 @@ function stopReasonLabel(run: RunForIssue) { if (stopReason === "process_lost") return "process lost"; if (stopReason === "unmanaged_background_task_stopped") return "unmanaged background task stopped"; - if (stopReason === "adapter_failed") return "adapter failed"; + if (stopReason === "adapter_failed") { + // Older runs have no recorded cause and keep the plain label. + const failureCause = normalizeRunFailureCause(run.failureCause); + return failureCause ? `adapter failed (${failureCause})` : "adapter failed"; + } if (stopReason === "completed") return timeoutText ? `completed (${timeoutText})` : "completed"; return timeoutText; diff --git a/ui/storybook/fixtures/paperclipData.ts b/ui/storybook/fixtures/paperclipData.ts index 4a3ab2ddd16..c10257e452a 100644 --- a/ui/storybook/fixtures/paperclipData.ts +++ b/ui/storybook/fixtures/paperclipData.ts @@ -1080,6 +1080,26 @@ export const storybookIssueRuns: RunForIssue[] = [ lastUsefulActionAt: null, nextAction: "Implement issue-management stories in the next heartbeat.", }, + { + runId: "run-storybook-adapter-cause", + status: "failed", + agentId: "agent-codex", + adapterType: "codex_local", + startedAt: recent(260).toISOString(), + finishedAt: recent(255).toISOString(), + createdAt: recent(260).toISOString(), + invocationSource: "scheduler", + errorCode: "adapter_failed", + failureCause: "40001", + usageJson: { costCents: 6, inputTokens: 2100, outputTokens: 140 }, + resultJson: { stopReason: "adapter_failed" }, + logBytes: 2400, + livenessState: "failed", + livenessReason: "The run failed before it produced concrete action evidence.", + continuationAttempt: 0, + lastUsefulActionAt: null, + nextAction: "Retry after the database conflict clears.", + }, ]; export const storybookApprovals: Approval[] = [