diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index affb9ac7a39..1bd9d7a81da 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -116,6 +116,11 @@ runtime test accepts the form’s prospective adapter selection before it is sav For a saved-agent test, omitting `environmentId` uses the agent’s saved override. Sending `environmentId: null` tests a change back to the instance default. +Saved-agent lifecycle verification checks managed account access before it +allocates a test execution environment. Missing defaults, revoked accounts, and +incompatible selections fail without creating a sandbox. The managed probe +checks account access again after allocation so changes during setup still apply. + ## On-demand usage limits The AI account detail has **Check usage**. It reads the selected account only diff --git a/server/src/__tests__/agent-harness-verification-preflight.test.ts b/server/src/__tests__/agent-harness-verification-preflight.test.ts new file mode 100644 index 00000000000..b03d955e428 --- /dev/null +++ b/server/src/__tests__/agent-harness-verification-preflight.test.ts @@ -0,0 +1,128 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, aiProviderDefaults, companies, companyMemberships, connectionGrants, createDb } from "@paperclipai/db"; +import type { AdapterEnvironmentTestResult } from "@paperclipai/adapter-utils"; +import { agentHarnessVerificationService } from "../services/agent-harness-verification.js"; +import * as environmentTests from "../services/agent-environment-test.js"; +import { aiConnectionService } from "../services/ai-connections.js"; +import { readAiConnectionConfigurationFailure } from "../services/ai-connection-configuration-failure.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +const realEnvironmentTests = environmentTests.agentEnvironmentTestService; + +describePostgres("saved harness account preflight", () => { + let database: Awaited>; + let db: ReturnType; + let home: string; + let agent: typeof agents.$inferSelect; + let userId: string; + let account: { connectionId: string; grantId: string }; + let fixtureNumber = 0; + const binding = { provider: "openrouter", method: "api_key", mode: "responsible_user" } as const; + const passed: AdapterEnvironmentTestResult = { + adapterType: "opencode_local", status: "pass", testedAt: new Date().toISOString(), + checks: [{ code: "opencode_hello_probe_passed", level: "info", message: "Fixture probe passed" }], + }; + const release = vi.fn<(status?: "released" | "failed") => Promise>(); + const allocate = vi.fn["resolveAdapterTestExecutionContext"]>(); + const probe = vi.fn["testManagedEnvironment"]>(); + const verify = () => agentHarnessVerificationService(db, {} as PluginWorkerManager).verify(agent); + + beforeAll(async () => { + home = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-harness-preflight-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_INSTANCE_ID", "harness-preflight-fixture"); + database = await startEmbeddedPostgresTestDatabase("paperclip-harness-preflight-db-"); + db = createDb(database.connectionString); + }, 90_000); + + beforeEach(async () => { + release.mockReset().mockResolvedValue(undefined); + allocate.mockReset().mockResolvedValue({ + executionTarget: null, environmentName: "Fixture execution target", fallbackChecks: [], release, + }); + probe.mockReset().mockResolvedValue(passed); + vi.spyOn(environmentTests, "agentEnvironmentTestService").mockImplementation((database, manager) => ({ + ...realEnvironmentTests(database, manager), + resolveAdapterTestEnvironmentId: vi.fn().mockResolvedValue(null), + resolveAdapterTestExecutionContext: allocate, + testManagedEnvironment: probe, + })); + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("Unexpected provider request")); + const companyId = randomUUID(); + userId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Harness preflight", issuePrefix: `HP${++fixtureNumber}`, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" }); + [agent] = await db.insert(agents).values({ + companyId, name: "Fixture agent", adapterType: "opencode_local", lifecycleState: "verifying", + adapterConfig: { model: "openrouter/anthropic/claude-sonnet-4.5" }, runtimeConfig: { aiConnection: binding }, + }).returning(); + account = await aiConnectionService(db).save(companyId, userId, { + provider: "openrouter", method: "api_key", ownership: "personal", name: "Fixture account", + apiKey: "fixture-preflight-key", allAgents: true, agentIds: [], + }, "fixture-preflight-key"); + await aiConnectionService(db).setDefault(companyId, userId, account.grantId); + }); + + afterEach(() => { vi.restoreAllMocks(); }); + afterAll(async () => { + await database?.cleanup(); + vi.unstubAllEnvs(); + if (home) await rm(home, { recursive: true, force: true }); + }); + + it("rejects a missing default before allocating a target", async () => { + await db.delete(aiProviderDefaults).where(eq(aiProviderDefaults.companyId, agent.companyId)); + const failure = await verify().catch(error => error); + expect(readAiConnectionConfigurationFailure(failure)).toBe("ai_connection_default_missing"); + expect(allocate).not.toHaveBeenCalled(); + expect(probe).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); + }); + + it("rejects a revoked account before allocating a target", async () => { + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId)); + await expect(verify()).rejects.toThrow("Reconnect or validate the selected AI account"); + expect(allocate).not.toHaveBeenCalled(); + expect(probe).not.toHaveBeenCalled(); + }); + + it("rejects incompatible model routing before allocating a target", async () => { + await db.update(agents).set({ adapterConfig: { model: "openai/gpt-5" } }).where(eq(agents.id, agent.id)); + await expect(verify()).rejects.toThrow("Select an AI connection compatible with this harness and model"); + expect(allocate).not.toHaveBeenCalled(); + }); + + it("runs a valid managed probe and releases its target", async () => { + await expect(verify()).resolves.toBe("complete"); + expect(allocate).toHaveBeenCalledOnce(); + expect(probe).toHaveBeenCalledOnce(); + expect(probe.mock.calls[0][1].config.env).toMatchObject({ OPENROUTER_API_KEY: "fixture-preflight-key" }); + expect(release).toHaveBeenCalledExactlyOnceWith("released"); + expect(globalThis.fetch).not.toHaveBeenCalled(); + }); + + it("checks account access again after allocation and releases on revocation", async () => { + allocate.mockImplementationOnce(async () => { + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId)); + return { executionTarget: null, environmentName: "Fixture execution target", fallbackChecks: [], release }; + }); + await expect(verify()).rejects.toThrow("Reconnect or validate the selected AI account"); + expect(allocate).toHaveBeenCalledOnce(); + expect(probe).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledExactlyOnceWith("failed"); + }); + + it("releases the target when the managed probe fails", async () => { + probe.mockRejectedValueOnce(new Error("Fixture probe failure")); + await expect(verify()).rejects.toThrow("Fixture probe failure"); + expect(release).toHaveBeenCalledExactlyOnceWith("failed"); + }); +}); diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.ts index 20f64c64390..c84194a6c78 100644 --- a/server/src/services/agent-harness-verification.ts +++ b/server/src/services/agent-harness-verification.ts @@ -6,6 +6,7 @@ import { isForbiddenConfigEnvKey, parseObject } from "@paperclipai/adapter-utils import { requireServerAdapter } from "../adapters/index.js"; import { agentEnvironmentTestService } from "./agent-environment-test.js"; import { aiConnectionRouterService } from "./ai-connection-router.js"; +import { aiConnectionService } from "./ai-connections.js"; import { withManagedAiProbe, stripAiAuthBindings } from "./ai-connection-runtime.js"; import { environmentService } from "./environments.js"; import { secretService } from "./secrets.js"; @@ -73,6 +74,11 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan { ...context, consumerType: "environment", consumerId: environmentId }); config = { ...config, env: { ...resolved.env, ...parseObject(config.env) } }; } + if (binding) await aiConnectionService(db).select({ + companyId: agent.companyId, userId: responsibleUserId, agentId: agent.id, + adapterType: agent.adapterType, model: config.model, runnerProvider: config.provider, + acpxAgent: config.acpxAgent, binding, + }); const target = await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId }); let status: "released" | "failed" = "failed"; try {