From d972a0955d89679a63fd2445bfebe8299d928f3a Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Nov 2026 01:02:42 -0600 Subject: [PATCH] fix(workspaces): carry the setup cause through the task query and redact before shortening - Select setupCommandFailure (and a scheduled retry's setup_failed predecessor's) in the task interruption query, and select the error text of an older setup_failed run that has the fixed command message. - Remove the run's registered secret values from the whole command and output before shortening them, at write and at read. - Parse older quoted and multiline commands. Co-Authored-By: Claude --- .../__tests__/task-interruption-query.test.ts | 46 +++++++++++++++++++ .../src/services/heartbeat/run-completion.ts | 2 +- .../task-interruption-presentation.test.ts | 22 +++++++++ .../task-interruption-presentation.ts | 45 ++++++++++++------ .../workspace-setup-command-failure.ts | 38 +++++++++++---- 5 files changed, 127 insertions(+), 26 deletions(-) diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index f4fbc476821..59d4bad1f43 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -353,6 +353,52 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => } finally { vi.unstubAllEnvs(); } }); + it("names a failed setup command on task detail and list reads, redacted with the run's registry (CF1)", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + try { + const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const secret = `private-setup-value-${randomUUID()}`; + const command = "bash ./scripts/provision-worktree.sh"; + const output = `bash: ./scripts/provision-worktree.sh: No such file or directory (${secret})`; + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "setup_failed", + error: `Execution workspace provision command "${command}" failed: ${output}`, + resultJson: { setupCommandFailure: { step: "workspace_provision", command, exitCode: 127, output } } }).where(eq(heartbeatRuns.id, run!.id)); + await createRunSecretRedactionRegistry(db).register(f.companyId, run!.id, secret); + const read = async () => [(await taskInterruptionPresentations(db, f.companyId, f.input)).get(f.input[0]!.id), + await taskInterruptionPresentation(db, f.input[0]!)]; + for (const presentation of await read()) { + expect(presentation).toMatchObject({ state: "failed", canRetry: true, setupFailure: { step: "workspace_provision", command, exitCode: 127 } }); + expect(presentation?.setupFailure?.message).toContain(`The project's workspace provision command "${command}" failed with exit code 127`); + expect(presentation?.setupFailure?.output).toContain("REDACTED"); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + // A run recorded before the structured cause existed is named from its fixed message shape. Its + // registered secret crosses the 300-character cut; it is removed whole before shortening. + const longOutput = `${secret}${"z".repeat(290)}`; + await db.update(heartbeatRuns).set({ resultJson: {}, error: `Execution workspace provision command "${command}" failed: ${longOutput}` }) + .where(eq(heartbeatRuns.id, run!.id)); + for (const presentation of await read()) { + expect(presentation?.setupFailure).toMatchObject({ step: "workspace_provision", command, exitCode: null }); + expect(presentation?.setupFailure?.output?.length).toBeLessThanOrEqual(300); + const text = JSON.stringify(presentation); + for (let index = 0; index + 8 <= secret.length; index += 1) expect(text).not.toContain(secret.slice(index, index + 8)); + } + // A transient retry shows its setup_failed predecessor's cause. + const predecessorId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: predecessorId, companyId: f.companyId, agentId: run!.agentId, status: "failed", + errorCode: "setup_failed", resultJson: { setupCommandFailure: { step: "workspace_provision", command, exitCode: 1, output } }, + createdAt: new Date(run!.createdAt.getTime() - 60_000) }); + await createRunSecretRedactionRegistry(db).register(f.companyId, predecessorId, secret); + await db.update(heartbeatRuns).set({ status: "scheduled_retry", retryOfRunId: predecessorId, errorCode: null, error: null, resultJson: {}, + scheduledRetryAt: new Date(Date.now() + 30_000), scheduledRetryAttempt: 1 }).where(eq(heartbeatRuns.id, run!.id)); + for (const presentation of await read()) { + expect(presentation?.setupFailure).toMatchObject({ command, exitCode: 1 }); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + } finally { vi.unstubAllEnvs(); } + }); + it.each(["decryption failure", "expanded message"])("keeps task reads available after provider display %s", async scenario => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); try { diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 0f5bb5c74c4..9e02b016a6d 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1521,7 +1521,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ...(connectionFailure ? { connectionFailure } : {}), ...(workspaceProviderFailure ? { workspaceProviderFailure } : {}), // Names the configured setup command that exited non-zero, for the task page. - ...setupCommandFailureResultJson(outerErr), + ...setupCommandFailureResultJson(outerErr, readFailureReportSecrets()), // A lease lost during setup surfaces as the thrown abort reason, or as // the signal's reason when a setup step threw a different error after it. ...controllerLeaseLossEvidence(outerErr), diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index fb7009172f3..c98c95f221c 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -238,6 +238,14 @@ describe("setup command failure presentation (CF1)", () => { expect(failure?.output).toMatch(/final line$/); }); + it("recognizes older quoted and multiline commands", () => { + for (const command of ['bash "./scripts/provision-worktree.sh"', "set -e\n./setup.sh"]) { + expect(projectTaskInterruption(issue, { run: { ...setupRun, resultJson: null, + error: `Execution workspace provision command "${command}" failed with exit code 2` }, agent, paused: false, approval: false })?.setupFailure) + .toMatchObject({ command, exitCode: 2 }); + } + }); + it("does not name setup for other failures or settled tasks", () => { expect(projectTaskInterruption(issue, { run: { ...setupRun, errorCode: "adapter_failed" }, agent, paused: false, approval: false })) .not.toHaveProperty("setupFailure"); @@ -432,3 +440,17 @@ describe("saved Retry held by dependencies", () => { expect(presentation?.retryWait?.message).toContain(text); }); }); + +describe("setup command failure evidence", () => { + it("removes a registered secret that crosses the output cut before shortening", async () => { + const { setupCommandFailureResultJson } = await import("./workspace-setup-command-failure.js"); + const secret = "registered-private-value-0123456789"; + const failure = Object.assign(new Error("setup failed"), { step: "workspace_provision", command: "./setup.sh", exitCode: 1, + // The cut keeps the last 299 characters; it lands inside the secret. + output: `${secret}${"y".repeat(299 - 10)}` }); + const recorded = setupCommandFailureResultJson(failure, [secret]); + expect(recorded).toMatchObject({ setupCommandFailure: { step: "workspace_provision", exitCode: 1 } }); + const text = JSON.stringify(recorded); + for (let index = 0; index + 8 <= secret.length; index += 1) expect(text).not.toContain(secret.slice(index, index + 8)); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 44cc1b600f1..2cbd5f2b52b 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,5 +1,5 @@ import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; -import { readSetupCommandFailure } from "./workspace-setup-command-failure.js"; +import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -198,10 +198,10 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio if (providerFailure) base.providerFailure = providerFailure; if (["failed", "scheduled_retry"].includes(run.status) && run.errorCode === "workspace_validation_failed" && typeof run.error === "string" && run.error.trim()) base.workspaceFailure = { message: run.error }; - if (["failed", "scheduled_retry"].includes(run.status) && run.errorCode === "setup_failed") { - const setupFailure = readSetupCommandFailure(run.resultJson, run.error); - if (setupFailure) base.setupFailure = setupFailure; - } + // A scheduled retry carries its setup_failed predecessor's cause; it has no error code of its own. + const setupFailure = run.status === "failed" && run.errorCode === "setup_failed" ? readSetupCommandFailure(run.resultJson, run.error) + : run.status === "scheduled_retry" ? readSetupCommandFailure(run.resultJson, null) : null; + if (setupFailure) base.setupFailure = setupFailure; const userAction = !issue.assigneeAgentId ? "assign_agent" : input.approval || run.errorCode === "native_provider_approval_required" ? "approval" : isAiAuthenticationBlocked(run) ? "reconnect" : run.errorCode === "native_provider_model_rejected" ? "choose_model" @@ -337,7 +337,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i agentAdapter: agents.adapterType, error: sql`case when ${heartbeatRuns.errorCode} = 'setup_failed' and ${heartbeatRuns.error} = 'Project is outside this low-trust boundary.' - or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' then ${heartbeatRuns.error} else null end`, + or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' + -- A setup command failure recorded before setupCommandFailure existed: only its fixed message shape. + or ${heartbeatRuns.errorCode} = 'setup_failed' + and ${heartbeatRuns.error} ~ '^(Execution workspace provision|Workspace seed|Runtime provision) command "' + then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, // Candidate only: the coordinator also pushes nextAttemptAt forward while @@ -357,6 +361,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i providerFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'providerFailure' is null and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, + // The run whose resultJson supplied setupCommandFailure (a scheduled retry shows its predecessor's). + setupFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'setupCommandFailure' is null + and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null + then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, resultJson: sql>`jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', 'stopReason', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'stopReason'`, 80)}, 'timeoutSource', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'timeoutSource'`, 80)}, @@ -378,7 +386,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i 'acknowledgementAuditId', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'nativeCancellation'->'acknowledgementAuditId'`, 160)}), 'providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' then (select predecessor.result_json->'providerFailure' from ${heartbeatRuns} predecessor - where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId}) end))`, + where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId}) end), + 'setupCommandFailure', coalesce(${heartbeatRuns.resultJson}->'setupCommandFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' + then (select predecessor.result_json->'setupCommandFailure' from ${heartbeatRuns} predecessor + where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId} + and predecessor.error_code = 'setup_failed') end))`, }).from(requested).innerJoinLateral(latestRuns, sql`true`).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, latestRuns.id))) .leftJoin(agents, and(eq(agents.companyId, heartbeatRuns.companyId), eq(agents.id, heartbeatRuns.agentId))) .orderBy(requestedIssueId), @@ -586,19 +598,21 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const disclosures = [...result.entries()].flatMap(([issueId, presentation]) => (presentation?.providerFailure || presentation?.workspaceFailure || presentation?.setupFailure) && presentation.observedRunId ? [{ id: presentation.observedRunId, providerFailureRunId: runMap.get(issueId)?.providerFailureRunId ?? presentation.observedRunId, + setupFailureRunId: runMap.get(issueId)?.setupFailureRunId ?? presentation.observedRunId, providerFailure: presentation.providerFailure, workspaceFailure: presentation.workspaceFailure, setupFailure: presentation.setupFailure, presentation }] : []); if (disclosures.length) { try { const [redacted, settings] = await Promise.all([ - createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.flatMap(({ id, providerFailureRunId, providerFailure, workspaceFailure, setupFailure }) => [ + createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.flatMap(({ id, providerFailureRunId, setupFailureRunId, providerFailure, workspaceFailure, setupFailure }) => [ { id: providerFailureRunId, providerFailure, workspaceFailure: undefined, setupFailure: undefined }, - { id, providerFailure: undefined, workspaceFailure, setupFailure }, + { id, providerFailure: undefined, workspaceFailure, setupFailure: undefined }, + { id: setupFailureRunId, providerFailure: undefined, workspaceFailure: undefined, setupFailure }, ])), instanceSettingsService(db).getGeneral(), ]); - const safe = disclosures.map((_, index) => ({ providerFailure: redacted[2 * index]?.providerFailure, - workspaceFailure: redacted[2 * index + 1]?.workspaceFailure, setupFailure: redacted[2 * index + 1]?.setupFailure })); + const safe = disclosures.map((_, index) => ({ providerFailure: redacted[3 * index]?.providerFailure, + workspaceFailure: redacted[3 * index + 1]?.workspaceFailure, setupFailure: redacted[3 * index + 2]?.setupFailure })); const safeMessage = (value: unknown) => typeof value === "string" ? [...redactCurrentUserText(value, { enabled: settings.censorUsernameInLogs })].slice(0, 4000).join("") : ""; disclosures.forEach(({ presentation }, index) => { @@ -616,10 +630,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i if (presentation.setupFailure) { const value = safe[index]?.setupFailure; const command = safeMessage(value?.command), message = safeMessage(value?.message); - if (value && command.trim() && message.trim()) { - const output = value.output == null ? null : safeMessage(value.output) || null; - presentation.setupFailure = { ...value, command, output, message }; - } else delete presentation.setupFailure; + const failure = value && command.trim() && message.trim() + ? boundSetupCommandFailure({ ...value, command, output: value.output == null ? null : safeMessage(value.output) || null, message }) + : null; + if (failure) presentation.setupFailure = failure; + else delete presentation.setupFailure; } }); } catch { diff --git a/server/src/services/workspace-setup-command-failure.ts b/server/src/services/workspace-setup-command-failure.ts index 42dfe21468a..1f63c33bb7a 100644 --- a/server/src/services/workspace-setup-command-failure.ts +++ b/server/src/services/workspace-setup-command-failure.ts @@ -19,24 +19,34 @@ const STEP_SETTINGS: Record = { workspace_runtime_provision: "The runtime provision command", }; // Messages written by workspace-runtime.ts before setup failures carried structured fields. -const LEGACY_MESSAGE = /^(Execution workspace provision|Workspace seed|Runtime provision) command "([^"\n]{1,2000})" failed(?:: ([\s\S]+)| with exit code (-?\d+))$/; +// The command is the text between the prefix and the last `" failed`, so quoted and multiline commands parse. +const LEGACY_MESSAGE = /^(Execution workspace provision|Workspace seed|Runtime provision) command "([\s\S]{1,2000})" failed(?: with exit code (-?\d+)|: ([\s\S]+))$/; const LEGACY_STEPS: Record = { "Execution workspace provision": "workspace_provision", "Workspace seed": "workspace_seed", "Runtime provision": "workspace_runtime_provision", }; -function bounded(value: string, max: number, keep: "head" | "tail"): string { - const clean = [...redactSensitiveText(value) - .replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f‪-‮⁦-⁩]/g, "").trim()]; +/** Whole registered secret values are removed before any truncation, so a cut can never keep a fragment. */ +function withoutSecrets(value: string, secrets: readonly string[]): string { + for (const secret of [...secrets].sort((left, right) => right.length - left.length)) { + if (secret.length >= 4) value = value.split(secret).join("[REDACTED]"); + } + return value; +} + +function bounded(value: string, max: number, keep: "head" | "tail", secrets: readonly string[] = []): string { + const clean = [...redactSensitiveText(withoutSecrets(value, secrets)) + .replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/g, "").trim()]; if (clean.length <= max) return clean.join(""); return keep === "head" ? `${clean.slice(0, max - 1).join("")}…` : `…${clean.slice(clean.length - (max - 1)).join("")}`; } -function normalize(step: SetupStep, command: string, exitCode: number | null, output: string | null): SetupCommandFailure | null { - const safeCommand = bounded(command, MAX_COMMAND_CHARS, "head"); +function normalize(step: SetupStep, command: string, exitCode: number | null, output: string | null, + secrets: readonly string[] = [], limits = { command: MAX_COMMAND_CHARS, output: MAX_OUTPUT_CHARS }): SetupCommandFailure | null { + const safeCommand = bounded(command, limits.command, "head", secrets); if (!safeCommand) return null; - const safeOutput = output ? bounded(output, MAX_OUTPUT_CHARS, "tail") || null : null; + const safeOutput = output ? bounded(output, limits.output, "tail", secrets) || null : null; const status = exitCode === null ? "failed" : `failed with exit code ${exitCode}`; return { step, command: safeCommand, exitCode, output: safeOutput, @@ -45,7 +55,7 @@ function normalize(step: SetupStep, command: string, exitCode: number | null, ou } /** Bounded, redacted result evidence for a run whose setup command exited non-zero. */ -export function setupCommandFailureResultJson(error: unknown): { setupCommandFailure: SetupCommandFailure } | Record { +export function setupCommandFailureResultJson(error: unknown, secrets: readonly string[] = []): { setupCommandFailure: SetupCommandFailure } | Record { let current: unknown = error; for (let depth = 0; depth < 6 && current; depth += 1) { // WorkspaceCommandExitError (workspace-runtime.ts), read by shape so this module stays light. @@ -53,7 +63,7 @@ export function setupCommandFailureResultJson(error: unknown): { setupCommandFai if (current instanceof Error && typeof value.step === "string" && SETUP_STEPS.has(value.step) && typeof value.command === "string") { const failure = normalize(value.step as SetupStep, value.command, typeof value.exitCode === "number" && Number.isInteger(value.exitCode) ? value.exitCode : null, - typeof value.output === "string" ? value.output : null); + typeof value.output === "string" ? value.output : null, secrets); return failure ? { setupCommandFailure: failure } : {}; } current = current instanceof Error ? current.cause : undefined; @@ -75,5 +85,13 @@ export function readSetupCommandFailure(resultJson: Record | nu if (typeof error !== "string") return null; const match = LEGACY_MESSAGE.exec(error.trim()); if (!match) return null; - return normalize(LEGACY_STEPS[match[1]!]!, match[2]!, match[4] === undefined ? null : Number(match[4]), match[3] ?? null); + // Older text was never shortened with its run's registered secrets removed, so it stays whole here. + // The task read removes those secrets first and then shortens it (boundSetupCommandFailure). + return normalize(LEGACY_STEPS[match[1]!]!, match[2]!, match[3] === undefined ? null : Number(match[3]), match[4] ?? null, + [], { command: Number.POSITIVE_INFINITY, output: Number.POSITIVE_INFINITY }); +} + +/** Shorten a projected setup failure after the run's registered secrets were removed from its whole text. */ +export function boundSetupCommandFailure(value: SetupCommandFailure): SetupCommandFailure | null { + return normalize(value.step, value.command, value.exitCode, value.output); }