diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 72b26df10c0..bb6d31c3cf6 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -124,6 +124,18 @@ export interface ExecutionReconciliation { export interface TaskInterruptionPresentation { providerFailure?: import("../provider-failure.generated.js").ProviderFailure; workspaceFailure?: { message: string }; + /** + * A configured workspace setup command exited non-zero before the agent + * started (`setup_failed`). Bounded and redacted: the step, the configured + * command, its exit status and a short tail of its output. + */ + setupFailure?: { + step: "workspace_provision" | "workspace_seed" | "workspace_runtime_provision"; + command: string; + exitCode: number | null; + output: string | null; + message: string; + }; /** A persisted Retry admission wait, normalized to public text by the server. */ retryWait?: { reason: string; message: string }; intentId: string | null; diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index f4fbc476821..89368dceef8 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -353,6 +353,56 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => } finally { vi.unstubAllEnvs(); } }); + it("names a failed setup command on task detail and list reads, redacted with the run's registry (CF1)", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + try { + const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const secret = `private-setup-value-${randomUUID()}`; + const command = "bash ./scripts/provision-worktree.sh"; + const output = `bash: ./scripts/provision-worktree.sh: No such file or directory (${secret})`; + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "setup_failed", + error: `Execution workspace provision command "${command}" failed: ${output}`, + resultJson: { setupCommandFailure: { step: "workspace_provision", command, exitCode: 127, output } } }).where(eq(heartbeatRuns.id, run!.id)); + await createRunSecretRedactionRegistry(db).register(f.companyId, run!.id, secret); + const read = async () => [(await taskInterruptionPresentations(db, f.companyId, f.input)).get(f.input[0]!.id), + await taskInterruptionPresentation(db, f.input[0]!)]; + for (const presentation of await read()) { + expect(presentation).toMatchObject({ state: "failed", canRetry: true, setupFailure: { step: "workspace_provision", command, exitCode: 127 } }); + expect(presentation?.setupFailure?.message).toContain(`The project's workspace provision command "${command}" failed with exit code 127`); + expect(presentation?.setupFailure?.output).toContain("REDACTED"); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + // A run recorded before the structured cause existed is named from its fixed message shape. Its + // registered secret crosses the 300-character cut; it is removed whole before shortening. + const longOutput = `${secret}${"z".repeat(290)}`; + await db.update(heartbeatRuns).set({ resultJson: {}, error: `Execution workspace provision command "${command}" failed: ${longOutput}` }) + .where(eq(heartbeatRuns.id, run!.id)); + for (const presentation of await read()) { + expect(presentation?.setupFailure).toMatchObject({ step: "workspace_provision", command, exitCode: null }); + expect(presentation?.setupFailure?.output?.length).toBeLessThanOrEqual(300); + const text = JSON.stringify(presentation); + for (let index = 0; index + 8 <= secret.length; index += 1) expect(text).not.toContain(secret.slice(index, index + 8)); + } + // A long older log still shows its final error line. + await db.update(heartbeatRuns).set({ error: `Execution workspace provision command "${command}" failed: ${"filler\n".repeat(800)}No such file or directory` }) + .where(eq(heartbeatRuns.id, run!.id)); + for (const presentation of await read()) expect(presentation?.setupFailure?.output).toMatch(/No such file or directory$/); + // A transient retry shows its setup_failed predecessor's cause. + const predecessorId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: predecessorId, companyId: f.companyId, agentId: run!.agentId, status: "failed", + errorCode: "setup_failed", resultJson: { setupCommandFailure: { step: "workspace_provision", command, exitCode: 1, output } }, + createdAt: new Date(run!.createdAt.getTime() - 60_000) }); + await createRunSecretRedactionRegistry(db).register(f.companyId, predecessorId, secret); + await db.update(heartbeatRuns).set({ status: "scheduled_retry", retryOfRunId: predecessorId, errorCode: null, error: null, resultJson: {}, + scheduledRetryAt: new Date(Date.now() + 30_000), scheduledRetryAttempt: 1 }).where(eq(heartbeatRuns.id, run!.id)); + for (const presentation of await read()) { + expect(presentation?.setupFailure).toMatchObject({ command, exitCode: 1 }); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + } finally { vi.unstubAllEnvs(); } + }); + it.each(["decryption failure", "expanded message"])("keeps task reads available after provider display %s", async scenario => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); try { diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 00a438cc03e..d47d65ec667 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -1,4 +1,5 @@ import { execFile, spawn } from "node:child_process"; +import { setupCommandFailureResultJson } from "../services/workspace-setup-command-failure.js"; import { createHash, randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; import fs from "node:fs/promises"; @@ -985,6 +986,37 @@ describe("realizeExecutionWorkspace", () => { } }); + it("names a failed project provision command with its exit status and output (CF1)", async () => { + const repoRoot = await createTempRepo(); + const config = buildExecutionWorkspaceAdapterConfig({ + agentConfig: {}, + projectPolicy: parseProjectExecutionWorkspacePolicy({ + enabled: true, + defaultMode: "isolated_workspace", + workspaceStrategy: { type: "git_worktree", baseRef: "main", provisionCommand: "bash ./scripts/missing-provisioner.sh" }, + }), + issueSettings: null, + mode: "isolated_workspace", + legacyUseProjectWorkspace: null, + }); + try { + const failure = await realizeExecutionWorkspace({ + base: { baseCwd: repoRoot, source: "project_primary", projectId: "project-1", workspaceId: "workspace-1", repoUrl: null, repoRef: "HEAD" }, + config, + issue: { id: "issue-1", identifier: "TEST-1", title: "Broken setup" }, + agent: { id: "agent-1", name: "Test agent", companyId: "company-1" }, + }).then(() => null, (error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(setupCommandFailureResultJson(failure)).toEqual({ setupCommandFailure: { + step: "workspace_provision", command: "bash ./scripts/missing-provisioner.sh", exitCode: 127, + output: expect.stringContaining("No such file or directory"), + message: expect.stringMatching(/^The project's workspace provision command "bash \.\/scripts\/missing-provisioner\.sh" failed with exit code 127: .*No such file or directory/), + } }); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } + }); + it.each([false, true])("realizes a plain Paperclip checkout without a local seed instance (image default env: %s)", async (defaultConfigInEnv) => { const repoRoot = await createTempRepo(); const paperclipHome = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-env-only-home-"))); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 9e536677957..0aae8d1725f 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -70,6 +70,7 @@ import type { getRunLogStore, RunLogHandle } from "../run-log-store.js"; import type { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js"; import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js"; import type { Db, agents } from "@paperclipai/db"; +import { setupCommandFailureResultJson } from "../workspace-setup-command-failure.js"; type Run = typeof heartbeatRuns.$inferSelect; type Agent = typeof agents.$inferSelect; @@ -1539,6 +1540,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...(connectionFailure ? { connectionFailure } : {}), ...(workspaceProviderFailure ? { workspaceProviderFailure } : {}), + // Names the configured setup command that exited non-zero, for the task page. + ...setupCommandFailureResultJson(outerErr, readFailureReportSecrets()), // A lease lost during setup surfaces as the thrown abort reason, or as // the signal's reason when a setup step threw a different error after it. ...controllerLeaseLossEvidence(outerErr), diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index f086b7cb4a2..c98c95f221c 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -195,6 +195,65 @@ it("preserves the actual workspace preparation error without changing retry auth .not.toHaveProperty("workspaceFailure"); }); +describe("setup command failure presentation (CF1)", () => { + const setupError = "Execution workspace provision command \"bash ./scripts/provision-worktree.sh\" failed: bash: ./scripts/provision-worktree.sh: No such file or directory"; + const setupRun = { ...run, errorCode: "setup_failed", error: setupError, resultJson: { setupCommandFailure: { + step: "workspace_provision", command: "bash ./scripts/provision-worktree.sh", exitCode: 127, + output: "bash: ./scripts/provision-worktree.sh: No such file or directory" } } }; + + it("names the broken setup command on the task without changing retry authority", () => { + const withCause = projectTaskInterruption(issue, { run: setupRun, agent, paused: false, approval: false }); + const withoutCause = projectTaskInterruption(issue, { run: { ...setupRun, error: "x", resultJson: null }, agent, paused: false, approval: false }); + expect(withCause).toMatchObject({ state: "failed", canRetry: true, userAction: null, setupFailure: { + step: "workspace_provision", command: "bash ./scripts/provision-worktree.sh", exitCode: 127, + message: "The project's workspace provision command \"bash ./scripts/provision-worktree.sh\" failed with exit code 127: bash: ./scripts/provision-worktree.sh: No such file or directory" } }); + const { setupFailure: _named, ...rest } = withCause!; + expect(rest).toEqual(withoutCause); + expect(withCause).not.toHaveProperty("workspaceFailure"); + }); + + it("recognizes a setup failure recorded before the structured cause existed", () => { + expect(projectTaskInterruption(issue, { run: { ...setupRun, resultJson: null }, agent, paused: false, approval: false })) + .toMatchObject({ state: "failed", setupFailure: { step: "workspace_provision", command: "bash ./scripts/provision-worktree.sh", + exitCode: null, output: "bash: ./scripts/provision-worktree.sh: No such file or directory" } }); + }); + + it("keeps a transient setup retry scheduled and only adds the name", () => { + const retrying = { ...setupRun, status: "scheduled_retry", scheduledRetryAt: new Date(Date.now() + 60_000), scheduledRetryReason: "transient_failure" }; + const named = projectTaskInterruption(issue, { run: retrying, agent, paused: false, approval: false }); + const unnamed = projectTaskInterruption(issue, { run: { ...retrying, error: "x", resultJson: null }, agent, paused: false, approval: false }); + expect(named?.setupFailure?.command).toBe("bash ./scripts/provision-worktree.sh"); + const { setupFailure: _named, ...rest } = named!; + expect(rest).toEqual(unnamed); + }); + + it("bounds the output tail and strips credentials from the command", () => { + const noisy = { ...setupRun, resultJson: { setupCommandFailure: { step: "workspace_provision", + command: "GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789 ./setup.sh --password hunter22", exitCode: 1, + output: `${"x".repeat(5_000)}\nfinal line` } } }; + const failure = projectTaskInterruption(issue, { run: noisy, agent, paused: false, approval: false })?.setupFailure; + expect(failure?.command).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz0123456789"); + expect(failure?.command).not.toContain("hunter22"); + expect(failure?.output?.length).toBeLessThanOrEqual(300); + expect(failure?.output).toMatch(/final line$/); + }); + + it("recognizes older quoted and multiline commands", () => { + for (const command of ['bash "./scripts/provision-worktree.sh"', "set -e\n./setup.sh"]) { + expect(projectTaskInterruption(issue, { run: { ...setupRun, resultJson: null, + error: `Execution workspace provision command "${command}" failed with exit code 2` }, agent, paused: false, approval: false })?.setupFailure) + .toMatchObject({ command, exitCode: 2 }); + } + }); + + it("does not name setup for other failures or settled tasks", () => { + expect(projectTaskInterruption(issue, { run: { ...setupRun, errorCode: "adapter_failed" }, agent, paused: false, approval: false })) + .not.toHaveProperty("setupFailure"); + expect(projectTaskInterruption({ ...issue, status: "done" }, { run: setupRun, agent, paused: false, approval: false })) + .not.toHaveProperty("setupFailure"); + }); +}); + function operatorStoppedRun() { return { ...run, status: "cancelled", runtimeMode: "native" as const, companyId: issue.companyId, nativeIssueId: issue.id, errorCode: "cancelled", resultJson: { @@ -381,3 +440,17 @@ describe("saved Retry held by dependencies", () => { expect(presentation?.retryWait?.message).toContain(text); }); }); + +describe("setup command failure evidence", () => { + it("removes a registered secret that crosses the output cut before shortening", async () => { + const { setupCommandFailureResultJson } = await import("./workspace-setup-command-failure.js"); + const secret = "registered-private-value-0123456789"; + const failure = Object.assign(new Error("setup failed"), { step: "workspace_provision", command: "./setup.sh", exitCode: 1, + // The cut keeps the last 299 characters; it lands inside the secret. + output: `${secret}${"y".repeat(299 - 10)}` }); + const recorded = setupCommandFailureResultJson(failure, [secret]); + expect(recorded).toMatchObject({ setupCommandFailure: { step: "workspace_provision", exitCode: 1 } }); + const text = JSON.stringify(recorded); + for (let index = 0; index + 8 <= secret.length; index += 1) expect(text).not.toContain(secret.slice(index, index + 8)); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 982783c98c3..5009f22e5f6 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,4 +1,5 @@ import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; +import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -197,6 +198,10 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio if (providerFailure) base.providerFailure = providerFailure; if (["failed", "scheduled_retry"].includes(run.status) && run.errorCode === "workspace_validation_failed" && typeof run.error === "string" && run.error.trim()) base.workspaceFailure = { message: run.error }; + // A scheduled retry carries its setup_failed predecessor's cause; it has no error code of its own. + const setupFailure = run.status === "failed" && run.errorCode === "setup_failed" ? readSetupCommandFailure(run.resultJson, run.error) + : run.status === "scheduled_retry" ? readSetupCommandFailure(run.resultJson, null) : null; + if (setupFailure) base.setupFailure = setupFailure; const userAction = !issue.assigneeAgentId ? "assign_agent" : input.approval || run.errorCode === "native_provider_approval_required" ? "approval" : isAiAuthenticationBlocked(run) ? "reconnect" : run.errorCode === "native_provider_model_rejected" ? "choose_model" @@ -332,7 +337,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i agentAdapter: agents.adapterType, error: sql`case when ${heartbeatRuns.errorCode} = 'setup_failed' and ${heartbeatRuns.error} = 'Project is outside this low-trust boundary.' - or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' then ${heartbeatRuns.error} else null end`, + or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' + -- A setup command failure recorded before setupCommandFailure existed: only its fixed message shape. + or ${heartbeatRuns.errorCode} = 'setup_failed' + and ${heartbeatRuns.error} ~ '^(Execution workspace provision|Workspace seed|Runtime provision) command "' + then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, // Candidate only: the coordinator also pushes nextAttemptAt forward while @@ -352,6 +361,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i providerFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'providerFailure' is null and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, + // The run whose resultJson supplied setupCommandFailure (a scheduled retry shows its predecessor's). + setupFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'setupCommandFailure' is null + and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null + then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, resultJson: sql>`jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', 'stopReason', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'stopReason'`, 80)}, 'timeoutSource', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'timeoutSource'`, 80)}, @@ -373,7 +386,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i 'acknowledgementAuditId', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'nativeCancellation'->'acknowledgementAuditId'`, 160)}), 'providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' then (select predecessor.result_json->'providerFailure' from ${heartbeatRuns} predecessor - where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId}) end))`, + where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId}) end), + 'setupCommandFailure', coalesce(${heartbeatRuns.resultJson}->'setupCommandFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' + then (select predecessor.result_json->'setupCommandFailure' from ${heartbeatRuns} predecessor + where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId} + and predecessor.error_code = 'setup_failed') end))`, }).from(requested).innerJoinLateral(latestRuns, sql`true`).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, latestRuns.id))) .leftJoin(agents, and(eq(agents.companyId, heartbeatRuns.companyId), eq(agents.id, heartbeatRuns.agentId))) .orderBy(requestedIssueId), @@ -579,20 +596,23 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i // workspace failure is the observed run's, while a scheduled retry's provider // failure may be its predecessor's. const disclosures = [...result.entries()].flatMap(([issueId, presentation]) => - (presentation?.providerFailure || presentation?.workspaceFailure) && presentation.observedRunId + (presentation?.providerFailure || presentation?.workspaceFailure || presentation?.setupFailure) && presentation.observedRunId ? [{ id: presentation.observedRunId, providerFailureRunId: runMap.get(issueId)?.providerFailureRunId ?? presentation.observedRunId, - providerFailure: presentation.providerFailure, workspaceFailure: presentation.workspaceFailure, presentation }] : []); + setupFailureRunId: runMap.get(issueId)?.setupFailureRunId ?? presentation.observedRunId, + providerFailure: presentation.providerFailure, workspaceFailure: presentation.workspaceFailure, + setupFailure: presentation.setupFailure, presentation }] : []); if (disclosures.length) { try { const [redacted, settings] = await Promise.all([ - createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.flatMap(({ id, providerFailureRunId, providerFailure, workspaceFailure }) => [ - { id: providerFailureRunId, providerFailure, workspaceFailure: undefined }, - { id, providerFailure: undefined, workspaceFailure }, + createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.flatMap(({ id, providerFailureRunId, setupFailureRunId, providerFailure, workspaceFailure, setupFailure }) => [ + { id: providerFailureRunId, providerFailure, workspaceFailure: undefined, setupFailure: undefined }, + { id, providerFailure: undefined, workspaceFailure, setupFailure: undefined }, + { id: setupFailureRunId, providerFailure: undefined, workspaceFailure: undefined, setupFailure }, ])), instanceSettingsService(db).getGeneral(), ]); - const safe = disclosures.map((_, index) => ({ providerFailure: redacted[2 * index]?.providerFailure, - workspaceFailure: redacted[2 * index + 1]?.workspaceFailure })); + const safe = disclosures.map((_, index) => ({ providerFailure: redacted[3 * index]?.providerFailure, + workspaceFailure: redacted[3 * index + 1]?.workspaceFailure, setupFailure: redacted[3 * index + 2]?.setupFailure })); const safeMessage = (value: unknown) => typeof value === "string" ? [...redactCurrentUserText(value, { enabled: settings.censorUsernameInLogs })].slice(0, 4000).join("") : ""; disclosures.forEach(({ presentation }, index) => { @@ -607,11 +627,23 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i if (message.trim()) presentation.workspaceFailure = { message }; else delete presentation.workspaceFailure; } + if (presentation.setupFailure) { + const value = safe[index]?.setupFailure; + // Redact the whole text, then shorten: the output keeps its final lines, not a head slice. + const whole = (text: unknown) => typeof text === "string" ? redactCurrentUserText(text, { enabled: settings.censorUsernameInLogs }) : ""; + const command = whole(value?.command), message = whole(value?.message); + const failure = value && command.trim() && message.trim() + ? boundSetupCommandFailure({ ...value, command, output: value.output == null ? null : whole(value.output) || null, message }) + : null; + if (failure) presentation.setupFailure = failure; + else delete presentation.setupFailure; + } }); } catch { for (const { presentation } of disclosures) { delete presentation.providerFailure; delete presentation.workspaceFailure; + delete presentation.setupFailure; } logger.warn({ companyId, runCount: disclosures.length }, "Could not redact run failure display; explanations omitted"); } diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 518f0898dfc..37eb1d413d0 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -3116,14 +3116,24 @@ function resolveRepoManagedWorkspaceCommand(command: string, repoRoot: string) { */ export class WorkspaceCommandExitError extends Error { readonly exitCode: number | null; + /** The configured setup step and command, so the task page can name the setting that broke. */ + readonly step: WorkspaceCommandPhase | null; + readonly command: string | null; + readonly output: string | null; - constructor(message: string, exitCode: number | null) { + constructor(message: string, exitCode: number | null, failure?: { step: WorkspaceCommandPhase; command: string; output: string }) { super(message); this.exitCode = exitCode; + this.step = failure?.step ?? null; + this.command = failure?.command ?? null; + this.output = failure?.output || null; } } +type WorkspaceCommandPhase = "workspace_provision" | "workspace_seed" | "workspace_runtime_provision" | "workspace_teardown"; + async function runWorkspaceCommand(input: { + phase?: WorkspaceCommandPhase; command: string; resolvedCommand?: string; cwd: string; @@ -3148,6 +3158,7 @@ async function runWorkspaceCommand(input: { ? `${input.label} failed: ${details}` : `${input.label} failed with exit code ${proc.code ?? -1}`, proc.code, + input.phase ? { step: input.phase, command: input.command, output: details } : undefined, ); } @@ -3276,7 +3287,7 @@ async function recordGitOperation( async function recordWorkspaceCommandOperation( recorder: WorkspaceOperationRecorder | null | undefined, input: { - phase: "workspace_provision" | "workspace_seed" | "workspace_runtime_provision" | "workspace_teardown"; + phase: WorkspaceCommandPhase; command: string; resolvedCommand?: string; cwd: string; @@ -3345,6 +3356,7 @@ async function recordWorkspaceCommandOperation( ? `${input.label} failed: ${details}` : `${input.label} failed with exit code ${code ?? -1}`, code, + { step: input.phase, command: input.command, output: details }, ); } diff --git a/server/src/services/workspace-setup-command-failure.ts b/server/src/services/workspace-setup-command-failure.ts new file mode 100644 index 00000000000..1f63c33bb7a --- /dev/null +++ b/server/src/services/workspace-setup-command-failure.ts @@ -0,0 +1,97 @@ +import type { TaskInterruptionPresentation } from "@paperclipai/shared"; +import { redactSensitiveText } from "../redaction.js"; + +/** + * A configured workspace setup command (a project or service setting) exited + * non-zero before the agent started. The run keeps its `setup_failed` + * classification and retry behavior; this bounded record only lets the task + * page name the setting that broke and what it printed. + */ +type SetupCommandFailure = NonNullable; +type SetupStep = SetupCommandFailure["step"]; + +const SETUP_STEPS: ReadonlySet = new Set(["workspace_provision", "workspace_seed", "workspace_runtime_provision"]); +const MAX_COMMAND_CHARS = 200; +const MAX_OUTPUT_CHARS = 300; +const STEP_SETTINGS: Record = { + workspace_provision: "The project's workspace provision command", + workspace_seed: "The workspace seed command", + workspace_runtime_provision: "The runtime provision command", +}; +// Messages written by workspace-runtime.ts before setup failures carried structured fields. +// The command is the text between the prefix and the last `" failed`, so quoted and multiline commands parse. +const LEGACY_MESSAGE = /^(Execution workspace provision|Workspace seed|Runtime provision) command "([\s\S]{1,2000})" failed(?: with exit code (-?\d+)|: ([\s\S]+))$/; +const LEGACY_STEPS: Record = { + "Execution workspace provision": "workspace_provision", + "Workspace seed": "workspace_seed", + "Runtime provision": "workspace_runtime_provision", +}; + +/** Whole registered secret values are removed before any truncation, so a cut can never keep a fragment. */ +function withoutSecrets(value: string, secrets: readonly string[]): string { + for (const secret of [...secrets].sort((left, right) => right.length - left.length)) { + if (secret.length >= 4) value = value.split(secret).join("[REDACTED]"); + } + return value; +} + +function bounded(value: string, max: number, keep: "head" | "tail", secrets: readonly string[] = []): string { + const clean = [...redactSensitiveText(withoutSecrets(value, secrets)) + .replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/g, "").trim()]; + if (clean.length <= max) return clean.join(""); + return keep === "head" ? `${clean.slice(0, max - 1).join("")}…` : `…${clean.slice(clean.length - (max - 1)).join("")}`; +} + +function normalize(step: SetupStep, command: string, exitCode: number | null, output: string | null, + secrets: readonly string[] = [], limits = { command: MAX_COMMAND_CHARS, output: MAX_OUTPUT_CHARS }): SetupCommandFailure | null { + const safeCommand = bounded(command, limits.command, "head", secrets); + if (!safeCommand) return null; + const safeOutput = output ? bounded(output, limits.output, "tail", secrets) || null : null; + const status = exitCode === null ? "failed" : `failed with exit code ${exitCode}`; + return { + step, command: safeCommand, exitCode, output: safeOutput, + message: `${STEP_SETTINGS[step]} "${safeCommand}" ${status}${safeOutput ? `: ${safeOutput}` : ""}`, + }; +} + +/** Bounded, redacted result evidence for a run whose setup command exited non-zero. */ +export function setupCommandFailureResultJson(error: unknown, secrets: readonly string[] = []): { setupCommandFailure: SetupCommandFailure } | Record { + let current: unknown = error; + for (let depth = 0; depth < 6 && current; depth += 1) { + // WorkspaceCommandExitError (workspace-runtime.ts), read by shape so this module stays light. + const value = current as { step?: unknown; command?: unknown; exitCode?: unknown; output?: unknown }; + if (current instanceof Error && typeof value.step === "string" && SETUP_STEPS.has(value.step) && typeof value.command === "string") { + const failure = normalize(value.step as SetupStep, value.command, + typeof value.exitCode === "number" && Number.isInteger(value.exitCode) ? value.exitCode : null, + typeof value.output === "string" ? value.output : null, secrets); + return failure ? { setupCommandFailure: failure } : {}; + } + current = current instanceof Error ? current.cause : undefined; + } + return {}; +} + +/** Read the recorded setup-command failure, or recognize one recorded before the structured field existed. */ +export function readSetupCommandFailure(resultJson: Record | null | undefined, error: unknown): SetupCommandFailure | null { + const stored = resultJson?.setupCommandFailure; + if (stored && typeof stored === "object" && !Array.isArray(stored)) { + const value = stored as Record; + if (typeof value.step === "string" && SETUP_STEPS.has(value.step) && typeof value.command === "string") { + return normalize(value.step as SetupStep, value.command, + typeof value.exitCode === "number" && Number.isInteger(value.exitCode) ? value.exitCode : null, + typeof value.output === "string" ? value.output : null); + } + } + if (typeof error !== "string") return null; + const match = LEGACY_MESSAGE.exec(error.trim()); + if (!match) return null; + // Older text was never shortened with its run's registered secrets removed, so it stays whole here. + // The task read removes those secrets first and then shortens it (boundSetupCommandFailure). + return normalize(LEGACY_STEPS[match[1]!]!, match[2]!, match[3] === undefined ? null : Number(match[3]), match[4] ?? null, + [], { command: Number.POSITIVE_INFINITY, output: Number.POSITIVE_INFINITY }); +} + +/** Shorten a projected setup failure after the run's registered secrets were removed from its whole text. */ +export function boundSetupCommandFailure(value: SetupCommandFailure): SetupCommandFailure | null { + return normalize(value.step, value.command, value.exitCode, value.output); +} diff --git a/tests/restart-chaos/config-faults.test.ts b/tests/restart-chaos/config-faults.test.ts index e1bbdc1ecfe..40d520a0690 100644 --- a/tests/restart-chaos/config-faults.test.ts +++ b/tests/restart-chaos/config-faults.test.ts @@ -105,6 +105,15 @@ describe("gradeConfigTask", () => { expect(grade.signature).toBe("CF1/provision_command:message_only_in_run_detail"); }); + it("passes CF1 when the task interruption names the failed setup command", () => { + const parked = snapshot(90_000, { runs: [failedRun("r1", 10_000, "Execution workspace provision command \"bash ./scripts/provision-worktree.sh\" failed")], + interruption: { ...generic, setupFailure: "The project's workspace provision command \"bash ./scripts/provision-worktree.sh\" failed with exit code 127" } }); + const grade = gradeConfigTask(record({}, { brokenSnapshots: [parked], parkedAt: 90_000, parkReason: "interruption", + restart: { kind: "K1", faultId: "f00", firedAt: 95_000, readyAt: 120_000, error: null }, afterRestartSnapshots: [parked], afterRestartParked: true, + repair: { at: 200_000, action: "clicked Retry", detail: "", error: null }, finalStatus: "done", doneAt: 400_000 }), options); + expect(grade.checks.find(check => check.name === "actionable_message")?.verdict).toBe("pass"); + }); + it("passes when the task names the setting, survives the restart and completes after repair", () => { const parked = snapshot(90_000, { runs: [failedRun("r1", 10_000, "x")], interruption: { ...generic, workspaceFailure: "The project workspace path is not available" } }); diff --git a/tests/restart-chaos/config-faults.ts b/tests/restart-chaos/config-faults.ts index e65a20928be..ea5fcc34cbf 100644 --- a/tests/restart-chaos/config-faults.ts +++ b/tests/restart-chaos/config-faults.ts @@ -168,7 +168,9 @@ export interface ConfigSnapshot { reachable: boolean; issueStatus: string | null; runs: Array<{ id: string; status: string; createdAt: number | null; startedAt: number | null; errorCode: string | null; error: string | null }>; - interruption: { state: string | null; canRetry: boolean; userAction: string | null; workspaceFailure: string | null; observedRunId: string | null } | null; + interruption: { state: string | null; canRetry: boolean; userAction: string | null; workspaceFailure: string | null; + /** The named setup command failure (`setup_failed`), as the task page shows it. */ + setupFailure?: string | null; observedRunId: string | null } | null; execution: { label: string | null; cause: string | null; nextAction: string | null } | null; /** Pending thread interactions on the task (connection cards and similar). */ cards: Array<{ id: string; kind: string; status: string; serviceSlug: string | null; agentSetup: boolean }>; @@ -182,6 +184,7 @@ export function taskLevelTexts(snapshot: ConfigSnapshot): string[] { const texts: string[] = []; const interruption = snapshot.interruption; if (interruption?.workspaceFailure) texts.push(interruption.workspaceFailure); + if (interruption?.setupFailure) texts.push(interruption.setupFailure); if (interruption?.userAction) texts.push(`userAction:${interruption.userAction}`); if (snapshot.execution?.label) texts.push(snapshot.execution.label); if (snapshot.execution?.nextAction) texts.push(snapshot.execution.nextAction); diff --git a/tests/restart-chaos/config-wave.ts b/tests/restart-chaos/config-wave.ts index e205c5c025c..117ea641530 100644 --- a/tests/restart-chaos/config-wave.ts +++ b/tests/restart-chaos/config-wave.ts @@ -233,7 +233,7 @@ async function snapshot(api: ChaosApi, issueId: string, agentId: string, errors: return { at, reachable: true, issueStatus: String(issue.status), runs: runSamples, interruption: interruption ? { state: interruption.state ?? null, canRetry: interruption.canRetry === true, userAction: interruption.userAction ?? null, - workspaceFailure: interruption.workspaceFailure?.message ?? null, observedRunId: interruption.observedRunId ?? null } : null, + workspaceFailure: interruption.workspaceFailure?.message ?? null, setupFailure: interruption.setupFailure?.message ?? null, observedRunId: interruption.observedRunId ?? null } : null, execution: projection ? { label: projection.label ?? null, cause: projection.cause ?? null, nextAction: projection.nextAction ?? null } : null, cards: (Array.isArray(interactions) ? interactions : []).map(card => ({ id: String(card.id), kind: String(card.kind), status: String(card.status), serviceSlug: card.payload?.serviceSlug ?? null, agentSetup: card.payload?.agentSetup === true })), diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsx index ad264030ef4..311816ce9d8 100644 --- a/ui/src/components/ExecutionBlockerNotice.test.tsx +++ b/ui/src/components/ExecutionBlockerNotice.test.tsx @@ -150,6 +150,18 @@ describe("ordinary task interruption", () => { await render({ ...failed, workspaceFailure, state: "working" }); expect(container.textContent).toBe(""); }); + it("names a failed workspace setup command and what to change, keeping Retry", async () => { + const setupFailure = { step: "workspace_provision" as const, command: "bash ./scripts/provision-worktree.sh", exitCode: 127, + output: "bash: ./scripts/provision-worktree.sh: No such file or directory", + message: "The project's workspace provision command \"bash ./scripts/provision-worktree.sh\" failed with exit code 127: bash: ./scripts/provision-worktree.sh: No such file or directory" }; + await render({ ...failed, setupFailure }); + expect(container.textContent).toContain("Couldn’t set up the task workspace."); + expect(container.textContent).toContain(setupFailure.message); + expect(container.textContent).toContain("Fix the command in the project’s workspace settings"); + expect(container.textContent).not.toContain("Couldn’t finish this attempt."); + await act(async () => container.querySelector("button")!.click()); + expect(retry).toHaveBeenCalledOnce(); + }); it("shows why an accepted Retry is waiting and keeps another attempt available", async () => { const retryWait = { reason: "remote_cleanup", message: "Retry is waiting for the previous environment to stop. Your message is saved." }; await render({ ...failed, retryWait }); diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsx index b4c12e44574..e082d512dec 100644 --- a/ui/src/components/ExecutionBlockerNotice.tsx +++ b/ui/src/components/ExecutionBlockerNotice.tsx @@ -123,10 +123,13 @@ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimi )}> {presentation.state === "failed" &&