From 95026970221bc72041330c8542abe31e2f80ca4b Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 10:15:48 -0500 Subject: [PATCH] Require authenticated Copilot model verification before agent adoption Co-Authored-By: Paperclip --- server/src/__tests__/ai-connections.test.ts | 57 +++++++++++++++++++ server/src/routes/agents.ts | 15 ++++- ui/src/components/new-agent/NewAgentSetup.tsx | 8 ++- ui/src/pages/NewAgent.test.tsx | 18 ++++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index f662f76e0c5..d8e112be74b 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -63,6 +63,63 @@ beforeAll(async () => { afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); describe("managed AI connections", () => { + it.each(["create", "hire", "edit"])("refuses unavailable Copilot models before %s persists configuration", async action => { + const { unprocessable } = await import("../errors.js"); + const { errorHandler } = await import("../middleware/index.js"); + const owner = `copilot-model-${action}`, id = randomUUID(); + const savedConfig = { provider: "acpx", acpxAgent: "copilot", model: "gpt-5.6-luna" }; + const aiConnection = { provider: "github", method: "api_key", mode: "responsible_user" } as const; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id, companyId, name: `Copilot ${action} fixture`, adapterType: "paperclip_runner", adapterConfig: savedConfig, runtimeConfig: { aiConnection } }); + const token = `github_pat_fixture_model_${action}`; + const account = await service.save(companyId, owner, { provider: "github", method: "api_key", ownership: "personal", name: `Model ${action}`, apiKey: token, agentIds: [id], allAgents: false }, token); + await service.setDefault(companyId, owner, account.grantId); + const [company] = await db.select().from(companies).where(eq(companies.id, companyId)); + await db.update(companies).set({ requireBoardApprovalForNewAgents: false }).where(eq(companies.id, companyId)); + const before = await db.select({ id: agents.id }).from(agents).where(eq(agents.companyId, companyId)); + const probe = vi.spyOn(copilotProbe, "probeCopilotExecutionTarget").mockRejectedValue(unprocessable("The selected Copilot model is unavailable for this account.", { code: "COPILOT_MODEL_UNAVAILABLE" })); + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: owner, companyIds: [companyId], isInstanceAdmin: true }; next(); }); + app.use("/api", agentRoutes(db)); app.use(errorHandler); + const config = { ...savedConfig, model: "unavailable-copilot-model" }; + try { + const response = action === "edit" + ? await request(app).patch(`/api/agents/${id}`).send({ adapterConfig: config }) + : await request(app).post(`/api/companies/${companyId}/${action === "hire" ? "agent-hires" : "agents"}`).send({ name: `Unavailable ${action}`, role: "general", adapterType: "paperclip_runner", adapterConfig: config, runtimeConfig: { aiConnection } }); + expect(response.status, JSON.stringify(response.body)).toBe(422); + expect(response.body).toMatchObject({ error: "The selected Copilot model is unavailable for this account." }); + expect(probe).toHaveBeenCalledExactlyOnceWith(token, null, "unavailable-copilot-model"); + expect((await db.select().from(agents).where(eq(agents.id, id)))[0].adapterConfig).toEqual(savedConfig); + expect(await db.select({ id: agents.id }).from(agents).where(eq(agents.companyId, companyId))).toHaveLength(before.length); + expect(JSON.stringify(response.body)).not.toContain(token); + } finally { + probe.mockRestore(); + await db.update(companies).set({ requireBoardApprovalForNewAgents: company.requireBoardApprovalForNewAgents }).where(eq(companies.id, companyId)); + } + }); + + it("verifies the exact available Copilot model before a model-only edit with an unchanged connection", async () => { + const { errorHandler } = await import("../middleware/index.js"); + const owner = "copilot-model-valid", id = randomUUID(); + const aiConnection = { provider: "github", method: "api_key", mode: "responsible_user" } as const; + const config = { provider: "acpx", acpxAgent: "copilot", model: "gpt-5.6-luna" }; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id, companyId, name: "Copilot available model", adapterType: "paperclip_runner", adapterConfig: { ...config, model: "previous-model" }, runtimeConfig: { aiConnection } }); + const token = "github_pat_fixture_model_valid"; + const account = await service.save(companyId, owner, { provider: "github", method: "api_key", ownership: "personal", name: "Valid model", apiKey: token, agentIds: [id], allAgents: false }, token); + await service.setDefault(companyId, owner, account.grantId); + const probe = vi.spyOn(copilotProbe, "probeCopilotExecutionTarget").mockResolvedValue({ status: "verified", version: "1.0.88", profileDigest: "fixture-profile", promptSent: false, models: [{ id: config.model, label: "GPT" }] }); + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: owner, companyIds: [companyId], isInstanceAdmin: true }; next(); }); + app.use("/api", agentRoutes(db)); app.use(errorHandler); + try { + const response = await request(app).patch(`/api/agents/${id}`).send({ adapterConfig: config }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(probe).toHaveBeenCalledExactlyOnceWith(token, null, config.model); + expect((await db.select().from(agents).where(eq(agents.id, id)))[0].adapterConfig).toMatchObject(config); + expect(response.body.runtimeConfig.aiConnection).toEqual(aiConnection); + } finally { probe.mockRestore(); } + }); it("passes the live sandbox worker manager to Copilot connection verification", async () => { const pluginWorkerManager = {} as PluginWorkerManager; const probe = vi.spyOn(copilotProbe, "probeCopilotConnection").mockResolvedValue({ diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index e30b9049e43..5470edddb9c 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3576,14 +3576,17 @@ export function agentRoutes( // Hiring is allowed before the responsible user has connected this // provider. Execution still resolves credentials and creates the normal // task connection request; compatibility and access denials stay errors. - if (newAgent && !test && binding.mode === "responsible_user" && error instanceof HttpError + if (newAgent && !test && binding.provider !== "github" && binding.mode === "responsible_user" && error instanceof HttpError && ["ai_connection_default_missing", "ai_connection_missing", "ai_connection_unavailable", "ai_connection_responsible_user_missing"].includes(String(asRecord(error.details)?.code))) { return null; } throw error; }); if (!selection) return undefined; - if (test) { + // Copilot model availability is account-specific. Verify the selected + // packaged runtime before creation/adoption even when a token is saved. + // This metadata-only path sends no model prompt. + if (test || binding.provider === "github") { const testEnvironmentId = await resolveAdapterTestEnvironmentId(companyId, environmentId); if (testEnvironmentId) await assertAdapterTestEnvironmentForCompany(companyId, testEnvironmentId); const target = await resolveAdapterTestExecutionContext({ companyId, adapterType, environmentId: testEnvironmentId }); @@ -3591,6 +3594,10 @@ export function agentRoutes( if (!target.executionTarget && target.fallbackChecks.length > 0) throw unprocessable("The agent environment is not available for adoption"); await withManagedAiProbe(db, { companyId, agentId, responsibleUserId: userId, adapterType, binding, config, allowUninstalledPersonal: newAgent, allowUninstalledShared, allowLegacyValidation: true }, async managed => { const result = await testManagedEnvironment(adapterType, { companyId, adapterType, config: managed.config, executionTarget: target.executionTarget, environmentName: target.environmentName }, binding, managed, agentId); + if (binding.provider === "github" && result.status === "fail") { + const failure = result.checks.find(check => check.level === "error"); + throw unprocessable(failure?.message ?? "Copilot metadata verification failed.", { code: failure?.code ?? "COPILOT_REQUEST_FAILED" }); + } if (result.status === "fail" || result.checks.some(check => check.code === ADAPTER_AUTH_MISSING_CHECK_CODE)) throw unprocessable("The selected AI connection failed validation in this agent’s environment. Run the agent test to see the failing checks.", { code: "ai_connection_validation_failed", checks: result.checks.filter(check => check.level === "error" || check.code === ADAPTER_AUTH_MISSING_CHECK_CODE).map(check => ({ code: check.code, level: check.level })), @@ -5829,7 +5836,9 @@ export function agentRoutes( const changed = JSON.stringify(nextAiBinding) !== JSON.stringify(existing.runtimeConfig.aiConnection); const aiConfig = (patchData.adapterConfig ?? existing.adapterConfig) as Record; if (nextAiBinding.mode !== "router" && !isAiConnectionCompatible(nextAiBinding, requestedAdapterType, aiConfig.model, aiConfig.provider, aiConfig.acpxAgent)) throw unprocessable("Select an AI connection compatible with the new harness and model"); - if (changed || (nextAiBinding.mode === "router" && requestedAdapterType !== existing.adapterType)) await validateManagedAgentBinding(req, existing.companyId, existing.id, requestedAdapterType, aiConfig, nextAiBinding, (patchData.defaultEnvironmentId !== undefined ? patchData.defaultEnvironmentId : existing.defaultEnvironmentId) as string | null, true); + const copilotConfigurationChanged = requestedAdapterType === "paperclip_runner" && aiConfig.provider === "acpx" && aiConfig.acpxAgent === "copilot" + && (touchesAdapterConfiguration || Object.prototype.hasOwnProperty.call(patchData, "defaultEnvironmentId")); + if (changed || copilotConfigurationChanged || (nextAiBinding.mode === "router" && requestedAdapterType !== existing.adapterType)) await validateManagedAgentBinding(req, existing.companyId, existing.id, requestedAdapterType, aiConfig, nextAiBinding, (patchData.defaultEnvironmentId !== undefined ? patchData.defaultEnvironmentId : existing.defaultEnvironmentId) as string | null, true); } if (requestedRuntimeConfig) patchData.runtimeConfig = requestedRuntimeConfig; if (touchesAdapterConfiguration || Object.prototype.hasOwnProperty.call(patchData, "defaultEnvironmentId")) { diff --git a/ui/src/components/new-agent/NewAgentSetup.tsx b/ui/src/components/new-agent/NewAgentSetup.tsx index 0ecc85d44bf..fbe0fe990f3 100644 --- a/ui/src/components/new-agent/NewAgentSetup.tsx +++ b/ui/src/components/new-agent/NewAgentSetup.tsx @@ -403,6 +403,8 @@ function Setup({ return config; } function preparedConfig(nextConnection = connection) { + if (brandType === "copilot_runtime" && !models.data?.some(candidate => candidate.id === model)) + throw new Error("Refresh Copilot models and select an available model before creating this agent."); if (isDot && !allowUnmeteredProvider) throw new Error("Acknowledge external provider billing before creating your Dot agent."); if (multiProvider && (!model.trim() || !model.includes("/"))) throw new Error("Choose or enter a model in provider/model format."); @@ -903,9 +905,9 @@ function Setup({ }} open={modelOpen} onOpenChange={setModelOpen} - allowDefault={!multiProvider} - required={multiProvider} - creatable + allowDefault={!multiProvider && brandType !== "copilot_runtime"} + required={multiProvider || brandType === "copilot_runtime"} + creatable={brandType !== "copilot_runtime"} groupByProvider={multiProvider && !connectionModels} preserveOrder={Boolean(connectionModels) || adapterCuratesModelOrder(brandType)} /> diff --git a/ui/src/pages/NewAgent.test.tsx b/ui/src/pages/NewAgent.test.tsx index 932c5a848f8..bfa81c4bf6f 100644 --- a/ui/src/pages/NewAgent.test.tsx +++ b/ui/src/pages/NewAgent.test.tsx @@ -201,6 +201,24 @@ afterEach(async () => { container.remove(); }); describe("New agent setup", () => { + it("refuses an undiscovered Copilot model before hiring with a saved connection", async () => { + api.adapterModels.mockResolvedValue([{ id: "gpt-5.6-luna", label: "GPT" }]); + await render("paperclip_runner", "copilot"); + await fill("Model", "unavailable-copilot-model"); + await click("Finish setup"); + expect(api.hire).not.toHaveBeenCalled(); + expect(container.textContent).toContain("select an available model"); + }); + it("preserves the authenticated Copilot model when hiring with a saved connection", async () => { + api.adapterModels.mockResolvedValue([{ id: "gpt-5.6-luna", label: "GPT" }]); + await render("paperclip_runner", "copilot"); + await fill("Model", "gpt-5.6-luna"); + await click("Finish setup"); + expect(api.hire).toHaveBeenCalledWith("company-1", expect.objectContaining({ + adapterType: "paperclip_runner", adapterConfig: expect.objectContaining({ provider: "acpx", acpxAgent: "copilot", model: "gpt-5.6-luna" }), + runtimeConfig: expect.objectContaining({ aiConnection: { provider: "github", method: "api_key", mode: "responsible_user" } }), + })); + }); it("discovers models after the first Copilot token becomes the responsible user's default", async () => { let defaultSaved = false; managedApi.setDefault.mockImplementation(async () => { defaultSaved = true; return {}; });