diff --git a/maps/agent-access-map/agents.md b/maps/agent-access-map/agents.md index 5988d4eace8..26e162cf032 100644 --- a/maps/agent-access-map/agents.md +++ b/maps/agent-access-map/agents.md @@ -132,6 +132,8 @@ API: `present` — representative operation IDs: - `POST /api/agents/{id}/pause`. - `POST /api/agents/{id}/resume`. - `POST /api/agents/{id}/terminate`. +- `POST /api/agents/{id}/lifecycle/retry`. +- `GET /api/agents/{id}` returns the safe failed phase/step and action in `lifecycleError`. Agent access: `partial` — REST / native `call_api` resume under agent_config:update change-grant authorization; pause and termination remain board-only. diff --git a/maps/feature-map/agents.md b/maps/feature-map/agents.md index 621e271ed13..44433068977 100644 --- a/maps/feature-map/agents.md +++ b/maps/feature-map/agents.md @@ -80,6 +80,12 @@ Automated: [instruction service](../../server/src/__tests__/agent-instructions-s Manual: Save a harmless instruction edit, reload, and verify its revision and effect in the next eligible run. Pause and resume the disposable agent; confirm pause prevents new work and does not erase previous runs. +For failed setup, use an unavailable model or invalid command on a disposable agent. +Confirm the profile names the failed verification step and gives a relevant action. +Check that it explains automatic retry and offers **Retry now**. Fix the configuration, +retry, and reload; the failure must disappear only after readiness commits. +Provider output and credentials must not appear in the lifecycle message. + An explicit runner change is a configuration change, with normal revisions and session invalidation. Incompatible settings must produce a field-specific error; selecting another runner must not silently discard them or change an active run's diff --git a/server/src/__tests__/agent-harness-verification.test.ts b/server/src/__tests__/agent-harness-verification.test.ts index 4728cdabd18..3d43bdd0a80 100644 --- a/server/src/__tests__/agent-harness-verification.test.ts +++ b/server/src/__tests__/agent-harness-verification.test.ts @@ -8,6 +8,22 @@ describe("saved harness verification", () => { adapterType: "test", status, testedAt: new Date().toISOString(), checks: codes.map(code => ({ code, level: "info", message: code })), }); + it.each([ + ["opencode_hello_probe_auth_required", "harness_auth_failed"], + ["opencode_hello_probe_model_unavailable", "harness_model_unavailable"], + ["opencode_command_unresolvable", "harness_command_unavailable"], + ["opencode_cwd_invalid", "harness_cwd_invalid"], + ["opencode_hello_probe_timed_out", "harness_probe_timed_out"], + ["opencode_hello_probe_unexpected_output", "harness_probe_unexpected_output"], + ["private provider error", "harness_test_failed"], + ])("classifies %s without keeping provider output", (checkCode, code) => { + try { + assertHarnessTestPassed(result("fail", [checkCode])); + expect.fail("Expected verification to fail"); + } catch (error) { + expect(error).toMatchObject({ code, message: "The harness test failed" }); + } + }); it.each([ ["fail", []], ["warn", ["claude_hello_probe_auth_required"]], ["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]], diff --git a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts index 06b7e246d49..39c7895c30b 100644 --- a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts @@ -5,6 +5,7 @@ import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../applica import { startAgentLifecycle } from "../index.js"; import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; import { logger } from "../../../middleware/logger.js"; +import { lifecycleFailureMessage } from "../application/failure.js"; const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3, lifecycleRequiredPluginIds: null, lifecycleError: null, @@ -66,6 +67,30 @@ describe("lifecycle readiness handoff", () => { }); describe("lifecycle failure diagnostics", () => { + it.each([ + ["host", "harness_model_unavailable", "Execution configuration check failed. The selected model is unavailable."], + ["plugin", "required_plugin_unavailable", "Execution configuration check: required plugin step failed. A required plugin is unavailable."], + ["select_plugins", "unknown private code", "Execution configuration check: finding required plugins failed. Paperclip could not complete this step."], + ["transition", "40P01", "Execution configuration check: saving progress failed. Paperclip could not complete this step."], + ] as const)("persists safe actionable errors for %s", async (stage, code, expected) => { + const { store, driver, worker } = fixture(); + const error = Object.assign(new Error("secret provider output"), { code }); + if (stage === "host") driver.runHost.mockRejectedValue(error); + if (stage === "plugin") driver.runPlugin.mockRejectedValue(error); + if (stage === "select_plugins") driver.requiredPluginIds.mockRejectedValue(error); + if (stage === "transition") store.change.mockRejectedValue(error); + await worker.process(agent.id); + expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date)); + expect(JSON.stringify(store.defer.mock.calls)).not.toContain("secret provider output"); + expect(JSON.stringify(store.defer.mock.calls)).not.toContain("unknown private code"); + await worker.stop(); + }); + + it("does not treat prototype properties as allowed failure codes", () => { + expect(lifecycleFailureMessage({ stage: "host", phase: "verifying" }, { code: "toString" })) + .toContain("Paperclip could not complete this step"); + }); + it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => { const { store, driver, report, worker } = fixture(); const error = new Error("private provider detail"); @@ -100,6 +125,7 @@ describe("lifecycle failure diagnostics", () => { it.each([ [{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"], [{ code: "harness_test_failed" }, "harness_test_failed"], + [{ code: "harness_auth_failed" }, "harness_auth_failed"], [{ cause: { code: "40P01", message: "private query" } }, "40P01"], [{ code: "private credential value" }, "unclassified"], ])("logs known codes without provider error contents", (details, failureCode) => { diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index fc84aa79212..725a4adf1c3 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -140,7 +140,7 @@ const support = await getEmbeddedPostgresTestSupport(); const work = worker(createLifecycleDriver(db, {} as never).runHost); await work.process(agent.id); expect(probe).toHaveBeenCalled(); - expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." }); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: expect.stringContaining("Execution configuration check failed. Authentication failed.") }); } finally { probe.mockRestore(); } }); diff --git a/server/src/modules/agent-lifecycle/application/failure.ts b/server/src/modules/agent-lifecycle/application/failure.ts new file mode 100644 index 00000000000..0c5da80e34b --- /dev/null +++ b/server/src/modules/agent-lifecycle/application/failure.ts @@ -0,0 +1,40 @@ +import type { LifecycleFailure } from "./ports.js"; + +// Only fixed text may leave this boundary. Provider errors can contain secrets. +const reasons = { + harness_test_failed: "The execution test did not pass. Open Configuration and run the agent test to inspect the failed checks.", + harness_auth_failed: "Authentication failed. Check the agent's AI connection or sign in to its execution tool, then retry.", + harness_model_unavailable: "The selected model is unavailable. Choose a model supported by the agent's AI connection in Configuration, then retry.", + harness_command_unavailable: "The execution command was not found. Check the command and its installation in the agent's environment, then retry.", + harness_cwd_invalid: "The working directory is unavailable. Check the working directory in Configuration, then retry.", + harness_probe_timed_out: "The execution test timed out. Paperclip will retry automatically. If this continues, run the agent test in Configuration.", + harness_probe_unexpected_output: "The execution test did not return the expected response. Run the agent test in Configuration to inspect the failed checks.", + test_environment_unavailable: "The test environment is unavailable. Check the agent's environment in Configuration. Paperclip will retry automatically.", + required_plugin_unavailable: "A required plugin is unavailable. Check the plugin's status in Settings. Paperclip will retry automatically.", + invalid_lifecycle_result: "A required plugin returned an invalid result. Check the plugin's status in Settings. Paperclip will retry automatically.", +} as const; + +const transientCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]); + +export function lifecycleFailureCode(error: unknown): string | undefined { + const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null; + return [candidate?.code, candidate?.cause?.code].find((code): code is string => + typeof code === "string" && (Object.hasOwn(reasons, code) || transientCodes.has(code))); +} + +export function lifecycleFailureMessage(context: LifecycleFailure, error: unknown): string { + const phases = { + preparing: "Agent preparation", verifying: "Execution configuration check", pausing: "Stopping agent execution", + resuming: "Starting agent resources", terminating: "Stopping agent execution", cleaning_up: "Agent resource cleanup", + }; + const phase = context.phase && Object.hasOwn(phases, context.phase) + ? phases[context.phase as keyof typeof phases] : "Agent setup"; + const step = context.stage === "plugin" ? `${phase}: required plugin step` + : context.stage === "select_plugins" ? `${phase}: finding required plugins` + : ["save_plugins", "complete_host", "complete_plugin", "transition"].includes(context.stage) + ? `${phase}: saving progress` : phase; + const code = lifecycleFailureCode(error); + const reason = code && Object.hasOwn(reasons, code) ? reasons[code as keyof typeof reasons] + : "Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator."; + return `${step} failed. ${reason}`; +} diff --git a/server/src/modules/agent-lifecycle/application/worker.ts b/server/src/modules/agent-lifecycle/application/worker.ts index 3b90b227673..3c362dcbb11 100644 --- a/server/src/modules/agent-lifecycle/application/worker.ts +++ b/server/src/modules/agent-lifecycle/application/worker.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js"; import { isLifecycleWorkPending } from "../domain/policy.js"; +import { lifecycleFailureMessage } from "./failure.js"; /** Database leases protect each attempt across server processes. */ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) { @@ -20,10 +21,13 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return; const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState, operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion }; - const runStep = (stage: LifecycleFailure["stage"], work: () => Promise, pluginId?: string) => - attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work); + let failureContext: LifecycleFailure = { ...context, stage: "host" }; + const runStep = (stage: LifecycleFailure["stage"], work: () => Promise, pluginId?: string) => { + const stepContext = { ...context, stage, ...(pluginId ? { pluginId } : {}) }; + return attempt(stepContext, work).catch(error => { failureContext = stepContext; throw error; }); + }; const renewal = setInterval(() => { - void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {}); + void attempt({ ...context, stage: "renew" }, () => store.renew(agent, owner, new Date())).catch(() => {}); }, 30_000); renewal.unref?.(); try { @@ -65,8 +69,9 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr async () => { await driver.onReady?.(changed); }).catch(() => {}); } if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return; - } catch { - if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue; + } catch (error) { + const message = lifecycleFailureMessage(failureContext, error); + if (!await runStep("defer", () => store.defer(agent, owner, message, new Date()))) continue; return; } finally { clearInterval(renewal); diff --git a/server/src/modules/agent-lifecycle/index.ts b/server/src/modules/agent-lifecycle/index.ts index ba8abce8c37..99fc03bb117 100644 --- a/server/src/modules/agent-lifecycle/index.ts +++ b/server/src/modules/agent-lifecycle/index.ts @@ -142,3 +142,5 @@ export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: L }, }; } + +export { lifecycleFailureCode } from "./application/failure.js"; diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.ts index 92e08a92cec..295397a445b 100644 --- a/server/src/services/agent-harness-verification.ts +++ b/server/src/services/agent-harness-verification.ts @@ -19,7 +19,17 @@ export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) { const authenticationFailed = result.checks.some(check => check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code)); const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe"))); - if (result.status === "fail" || authenticationFailed || incomplete) throw Object.assign(new Error("The harness test failed"), { code: "harness_test_failed" }); + if (result.status === "fail" || authenticationFailed || incomplete) { + const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix)); + const code = authenticationFailed ? "harness_auth_failed" + : has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable" + : has("_command_unresolvable") ? "harness_command_unavailable" + : has("_cwd_invalid") ? "harness_cwd_invalid" + : has("_hello_probe_timed_out") ? "harness_probe_timed_out" + : has("_hello_probe_unexpected_output") ? "harness_probe_unexpected_output" + : "harness_test_failed"; + throw Object.assign(new Error("The harness test failed"), { code }); + } } export function agentHarnessVerificationService(db: Db, manager: PluginWorkerManager) { diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 25ef5e58738..a69ae1d9120 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -16,11 +16,7 @@ export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle export function createAgentLifecycleEffects(hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects { return { reportFailure(context, error) { - // Only known codes are safe; provider errors can contain credentials or private data. - const allowedCodes = new Set(["required_plugin_unavailable", "invalid_lifecycle_result", "harness_test_failed", - "test_environment_unavailable", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]); - const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null; - const code = [candidate?.code, candidate?.cause?.code].find(value => typeof value === "string" && allowedCodes.has(value)); + const code = lifecycle.lifecycleFailureCode(error); logger.warn({ ...context, failureCode: code ?? "unclassified" }, "Agent lifecycle operation failed"); }, transaction: withAccountingTransaction, diff --git a/ui/src/components/AgentLifecycleStatus.test.tsx b/ui/src/components/AgentLifecycleStatus.test.tsx index 8e4467f2f46..e2fe7baf497 100644 --- a/ui/src/components/AgentLifecycleStatus.test.tsx +++ b/ui/src/components/AgentLifecycleStatus.test.tsx @@ -34,7 +34,9 @@ it("retries a failed step once while pending and removes the failure after recov try { await act(async () => root.render()); expect(container.textContent).toContain(agent.lifecycleError); - expect(container.textContent).toContain("Termination is not complete"); + expect(container.textContent).toContain("Agent cleanup failed"); + expect(container.textContent).toContain("Paperclip retries automatically"); + expect(container.textContent).not.toContain("Required resources are being removed"); await act(async () => container.querySelector("button")!.click()); expect(retry).toHaveBeenCalledOnce(); await act(async () => root.render()); diff --git a/ui/src/components/AgentLifecycleStatus.tsx b/ui/src/components/AgentLifecycleStatus.tsx index 29a6984da77..689a54b8314 100644 --- a/ui/src/components/AgentLifecycleStatus.tsx +++ b/ui/src/components/AgentLifecycleStatus.tsx @@ -15,6 +15,12 @@ const phases = { cleaning_up: ["Cleaning up agent", "Required resources are being removed. Termination is not complete."], } satisfies Partial>; +const failedPhases = { + preparing: "Agent preparation failed", verifying: "Agent verification failed", + pausing: "Agent pause failed", resuming: "Agent resume failed", + terminating: "Agent termination failed", cleaning_up: "Agent cleanup failed", +}; + function pendingPhase(state?: AgentLifecycleState) { return state && state in phases ? phases[state as keyof typeof phases] : undefined; } @@ -48,15 +54,16 @@ export function AgentLifecycleStatus({ agent, onRetry, retryPending, refreshErro {phase &&
{!agent.lifecycleError && - {phase[1]} + {!agent.lifecycleError && {phase[1]}}
} {phase && agent.lifecycleError &&
{agent.lifecycleError}
} + {phase && agent.lifecycleError &&

Paperclip retries automatically; Retry now checks again immediately.

} ; }