From bc4cad3ac837dfcc7aeeea8cfc86452be76800c6 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 6 Nov 2026 11:37:55 -0800 Subject: [PATCH] fix(environments): report an undescribable plugin Runner as a failed check (#19172) Return a retryable environment check when a plugin Runner cannot be described. Preserve stable Runner revision mismatch checks and preconfigured Runner-home handling. Validation: 108 focused tests pass. All 53 applicable checks pass, with three intentional skips. Greptile scored the exact head 5/5 and no review threads remain unresolved. Co-Authored-By: Paperclip --- .../agent-environment-test-status.test.ts | 33 +++++++++++++++++++ server/src/services/agent-environment-test.ts | 8 +++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/server/src/services/agent-environment-test-status.test.ts b/server/src/services/agent-environment-test-status.test.ts index 976b456c9a3..2991000c5a2 100644 --- a/server/src/services/agent-environment-test-status.test.ts +++ b/server/src/services/agent-environment-test-status.test.ts @@ -3,6 +3,7 @@ import * as executionTargets from "@paperclipai/adapter-utils/execution-target"; import type { AdapterEnvironmentTestContext } from "@paperclipai/adapter-utils"; import type { AdapterEnvironmentTestResult } from "@paperclipai/shared"; import type { Db } from "@paperclipai/db"; +import { environmentTaskErrorCodeData, JsonRpcCallError, PluginEnvironmentTaskError } from "@paperclipai/plugin-sdk"; const { runtimeProbe, claudeProbe, verifyKey, markAuthFailed } = vi.hoisted(() => ({ runtimeProbe: vi.fn(), claudeProbe: vi.fn(), verifyKey: vi.fn(), markAuthFailed: vi.fn(), @@ -11,6 +12,7 @@ vi.mock("../adapters/index.js", () => ({ requireServerAdapter: (type: string) => vi.mock("./ai-api-key-test.js", () => ({ validateAiApiKey: verifyKey })); vi.mock("./ai-connections.js", async original => ({ ...await original(), aiConnectionService: () => ({ markAuthenticationFailed: markAuthFailed }) })); import { agentEnvironmentTestService } from "./agent-environment-test.js"; +import { assertHarnessTestPassed } from "./agent-harness-verification.js"; const checkedAt = new Date(0).toISOString(); function readiness(): AdapterEnvironmentTestResult { @@ -136,6 +138,37 @@ describe("managed Runner connection test status", () => { checks: [{ code: "claude_hello_probe_passed", level: "info", message: "Hello completed with an unresolved warning." }] }); expect((await testConnection()).status).toBe("warn"); }); + it("reports a plugin task that cannot describe its Runner as a retryable failed check", async () => { + verifiedClaude(); + const runnerTask = vi.fn().mockRejectedValue(new Error("Plugin-provided Runner execution operation unavailable; reconcile the same task before retrying")); + const result = await testConnection({ executionTarget: { ...context.executionTarget!, runnerTask } as AdapterEnvironmentTestContext["executionTarget"] }); + expect(runnerTask).toHaveBeenCalledExactlyOnceWith({ kind: "describe" }); + expect(result.status).toBe("fail"); + expect(result.checks).toEqual([expect.objectContaining({ code: "runner_environment_unavailable", level: "error", retryable: true })]); + expect(JSON.stringify(result)).not.toContain("reconcile the same task"); + expect(() => assertHarnessTestPassed(result)).toThrow(expect.objectContaining({ code: "test_environment_unavailable" })); + expect(runtimeProbe).not.toHaveBeenCalled(); + expect(claudeProbe).not.toHaveBeenCalled(); + expect(markAuthFailed).not.toHaveBeenCalled(); + }); + it("still rejects a plugin task that describes no project authentication check", async () => { + verifiedClaude(); + const runnerTask = vi.fn().mockResolvedValue({ kind: "description", taskId: "task-1", runner: { version: "1", digest: `sha256:${"0".repeat(64)}` } }); + await expect(testConnection({ executionTarget: { ...context.executionTarget!, runnerTask } as AdapterEnvironmentTestContext["executionTarget"] })) + .rejects.toMatchObject({ status: 422, details: { code: "ai_connection_incompatible" } }); + expect(runtimeProbe).not.toHaveBeenCalled(); + }); + it("keeps the stable Runner revision mismatch result when plugin description fails", async () => { + verifiedClaude(); + const refusal = new JsonRpcCallError({ code: -32002, message: "The installed Runner does not match the expected revision", + data: environmentTaskErrorCodeData(new PluginEnvironmentTaskError("runner_revision_mismatch")) }); + const runnerTask = vi.fn().mockRejectedValue(refusal); + const result = await testConnection({ executionTarget: { ...context.executionTarget!, runnerTask } as AdapterEnvironmentTestContext["executionTarget"] }); + expect(result).toMatchObject({ status: "fail", checks: [{ code: "runner_revision_mismatch", level: "error", retryable: true }] }); + expect(() => assertHarnessTestPassed(result)).toThrow(expect.objectContaining({ code: "runner_revision_mismatch" })); + expect(runtimeProbe).not.toHaveBeenCalled(); + expect(markAuthFailed).not.toHaveBeenCalled(); + }); it("keeps actionable warnings after a successful hello", async () => { vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "managed-fixture"); verifiedClaude(); const result = readiness(); result.checks.push({ code: "provider_configuration_warning", level: "warn", message: "Check the selected provider configuration." }); runtimeProbe.mockResolvedValue(result); diff --git a/server/src/services/agent-environment-test.ts b/server/src/services/agent-environment-test.ts index ade8bb8709b..38cb939f11c 100644 --- a/server/src/services/agent-environment-test.ts +++ b/server/src/services/agent-environment-test.ts @@ -581,10 +581,12 @@ export function agentEnvironmentTestService(db: Db, pluginWorkerManager?: Plugin && target?.kind === "remote" && target.transport === "sandbox" && target.runnerTask) { return pluginRunnerAiConnectionRequiredTestResult(adapterType, true); } - try { await assertManagedAiProjectAuth(context.config, binding.provider, context.executionTarget); } + try { await assertManagedAiProjectAuth(context.config, binding.provider, target); } catch (error) { - if (!isRunnerRevisionMismatch(error)) throw error; - return runnerRevisionMismatchTestResult(adapterType); + if (isRunnerRevisionMismatch(error)) return runnerRevisionMismatchTestResult(adapterType); + if (error instanceof HttpError || target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask) throw error; + return { adapterType, status: "fail", testedAt: new Date().toISOString(), checks: [{ code: "runner_environment_unavailable", level: "error", retryable: true, + message: "The selected plugin environment could not describe its Runner. Check the environment's computer and plugin status, then retry." }] }; } const result = adapterType === "paperclip_runner" && target?.kind === "remote" && target.transport === "sandbox" && target.runnerTask ? await probePluginRunnerEnvironment(context, { identity: managed.identity, accountIdentity: managed.sessionIdentity,