mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 20:21:50 +01:00
feat(connections): add experimental task-pinned AI routing
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
806af230b6
commit
c2efe3e92a
51 files changed
+1763
-60
No files matched your search
@@ -0,0 +1,219 @@
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { createDb, companies, agents, companyMemberships, connectionGrants, plugins, aiConnectionRouterCursors, aiConnectionTaskPins, toolConnections, companySecrets } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres";
|
||||
import type { AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { aiConnectionRouterService, AiConnectionPoolExhausted, poolMemberRuntimeConfig, applyAiConnectionRouterTaskSettings } from "../services/ai-connection-router.js";
|
||||
import { projectPaperclipRunnerTaskConfig, resolvePaperclipRunnerNativeProviderInput } from "../services/native-runtime/provider-profile.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { managedAiSessionFingerprintConfig, prepareManagedAiRuntime } from "../services/ai-connection-runtime.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { aiConnectionRoutes } from "../routes/ai-connections.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let home: string;
|
||||
const companyId = randomUUID(), otherCompanyId = randomUUID(), agentId = randomUUID(), secondAgentId = randomUUID(), pluginId = randomUUID();
|
||||
const pluginKey = "fixture.ai-router";
|
||||
const manifest: PaperclipPluginManifestV1 = { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Fixture router", description: "Fixture", author: "Tests", categories: ["connector"], capabilities: ["ai.connections.route"], entrypoints: { worker: "worker.js" } };
|
||||
let members: AiConnectionPoolMember[];
|
||||
const proposal = vi.fn(async (_id: string, _method: string, request: AiConnectionRouterRequest): Promise<AiConnectionRouterResult> => {
|
||||
const next = (request.memberOrder.indexOf(request.lastMemberId ?? "") + 1) % request.memberOrder.length;
|
||||
const order = request.pinnedMemberId ? [request.pinnedMemberId] : [...request.memberOrder.slice(next), ...request.memberOrder.slice(0, next)];
|
||||
const selected = order.find(id => request.candidates.some(candidate => candidate.member.id === id));
|
||||
return selected ? { kind: "selected", memberId: selected } : { kind: "unavailable", skipped: {} };
|
||||
});
|
||||
const worker = { isRunning: () => true, call: proposal } as unknown as PluginWorkerManager;
|
||||
const service = () => aiConnectionRouterService(db, worker);
|
||||
const resolve = (poolId: string, taskKey: string, extra = {}) => service().resolve({ companyId, poolId, agentId, userId: "alice", adapterType: "paperclip_runner", taskKey, ...extra });
|
||||
const makePool = async (config = {}) => service().save(pluginKey, { companyId, config: { name: randomUUID(), enabled: true, mode: "round_robin", thresholdPercent: 90, members, ...config } }, "alice");
|
||||
beforeAll(async () => {
|
||||
home = await mkdtemp(path.join(os.tmpdir(), "paperclip-router-tests-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "router-fixture");
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-router-db-"); db = createDb(database.connectionString);
|
||||
await db.insert(companies).values([{ id: companyId, name: "Router fixture", issuePrefix: "RTF" }, { id: otherCompanyId, name: "Other", issuePrefix: "RTO" }]);
|
||||
await db.insert(agents).values([agentId, secondAgentId].map(id => ({ id, companyId, name: "Router agent", adapterType: "paperclip_runner" })));
|
||||
await db.insert(companyMemberships).values(["alice", "bob"].map(principalId => ({ companyId, principalId, principalType: "user", status: "active", membershipRole: "member" })));
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: pluginKey, version: "0.1.0", manifestJson: manifest, status: "ready" });
|
||||
const accounts = aiConnectionService(db);
|
||||
const codex = await accounts.save(companyId, "alice", { provider: "openai", method: "api_key", ownership: "personal", name: "Codex", apiKey: "fixture", agentIds: [], allAgents: true }, "fixture-codex");
|
||||
const claude = await accounts.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "personal", name: "Claude", loginSessionId: "fixture", agentIds: [], allAgents: true }, "fixture-claude");
|
||||
members = [
|
||||
{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", ...codex }, profile: { provider: "codex", model: "gpt-5.6-sol", effort: "low" } },
|
||||
{ id: randomUUID(), binding: { mode: "delegated", provider: "anthropic", method: "subscription", ...claude }, profile: { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-5" } },
|
||||
];
|
||||
}, 90000);
|
||||
afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); });
|
||||
|
||||
describe("durable, authorized connection routing", () => {
|
||||
it("defaults off and does not expose selectable pools before opt-in", async () => {
|
||||
expect((await instanceSettingsService(db).getExperimental()).enableAiConnectionRouters).toBe(false);
|
||||
await expect(makePool()).rejects.toThrow("Experimental");
|
||||
expect(await service().selectable(companyId, "alice")).toEqual([]);
|
||||
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true });
|
||||
});
|
||||
it("requires company connection management authority on pool configuration routes", async () => {
|
||||
const app = express(); app.use(express.json());
|
||||
let actor = { type: "board", userId: "alice", source: "session", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole: "owner" }] };
|
||||
app.use((req, _res, next) => { req.actor = actor as typeof req.actor; next(); }); app.use("/api", aiConnectionRoutes(db)); app.use(errorHandler);
|
||||
const pool = await makePool(); const url = `/api/companies/${companyId}/ai-connection-pools`;
|
||||
expect((await request(app).get(url)).status).toBe(200);
|
||||
expect((await request(app).get(`/api/companies/${otherCompanyId}/ai-connection-pools`)).status).toBe(403);
|
||||
actor = { ...actor, memberships: [{ companyId, status: "active", membershipRole: "viewer" }] };
|
||||
expect((await request(app).get(url)).status).toBe(403);
|
||||
expect((await request(app).post(url).send({ pluginKey, config: pool })).status).toBe(403);
|
||||
expect((await request(app).get(`${url}/${pool.id}/inspection`)).status).toBe(403);
|
||||
actor = { type: "agent", agentId, companyId, source: "agent_jwt" } as unknown as typeof actor;
|
||||
expect((await request(app).get(url)).status).toBe(403);
|
||||
});
|
||||
it("rotates new tasks across qualified backends and retains pins on turns, resets and service restart", async () => {
|
||||
const pool = await makePool();
|
||||
expect((await resolve(pool.id, "one")).memberId).toBe(members[0]!.id);
|
||||
expect((await resolve(pool.id, "two")).memberId).toBe(members[1]!.id);
|
||||
expect((await resolve(pool.id, "one", { overrides: { model: "claude-sonnet-5", modelReasoningEffort: "high" } })).memberId).toBe(members[0]!.id);
|
||||
expect((await aiConnectionRouterService(db, worker).resolve({ companyId, poolId: pool.id, agentId, userId: "alice", adapterType: "paperclip_runner", taskKey: "one" })).memberId).toBe(members[0]!.id);
|
||||
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(eq(aiConnectionRouterCursors.poolId, pool.id)); expect(cursor?.version).toBe(2);
|
||||
});
|
||||
it("commits one allocation for duplicate starts and serializes concurrent tasks", async () => {
|
||||
const pool = await makePool();
|
||||
const same = await Promise.all(Array.from({ length: 8 }, () => resolve(pool.id, "duplicate")));
|
||||
expect(new Set(same.map(s => s.memberId)).size).toBe(1);
|
||||
const many = await Promise.all(Array.from({ length: 8 }, (_, n) => resolve(pool.id, `concurrent-${n}`)));
|
||||
expect(many.filter(s => s.memberId === members[0]!.id)).toHaveLength(4);
|
||||
const pins = await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, pool.id)); expect(pins).toHaveLength(9);
|
||||
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(eq(aiConnectionRouterCursors.poolId, pool.id)); expect(cursor?.version).toBe(9);
|
||||
});
|
||||
it("rolls back both pin and cursor on a failure during commit", async () => {
|
||||
const pool = await makePool();
|
||||
await db.execute(`CREATE FUNCTION fixture_router_crash() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'fixture commit crash'; END $$`);
|
||||
await db.execute(`CREATE TRIGGER fixture_router_crash BEFORE UPDATE ON ai_connection_router_cursors FOR EACH ROW EXECUTE FUNCTION fixture_router_crash()`);
|
||||
try { await expect(resolve(pool.id, "crash")).rejects.toThrow(); } finally { await db.execute(`DROP TRIGGER fixture_router_crash ON ai_connection_router_cursors`); await db.execute(`DROP FUNCTION fixture_router_crash()`); }
|
||||
expect(await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, pool.id))).toHaveLength(0);
|
||||
const selected = await resolve(pool.id, "crash"); expect(selected.memberId).toBe(members[0]!.id);
|
||||
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(eq(aiConnectionRouterCursors.poolId, pool.id)); expect(cursor?.version).toBe(1);
|
||||
});
|
||||
it("rechecks config revisions after a proposal and preserves removed member snapshots", async () => {
|
||||
const pool = await makePool(); const old = await resolve(pool.id, "existing");
|
||||
const saved = await service().save(pluginKey, { companyId, id: pool.id, expectedRevision: pool.revision, config: { name: "Edited", enabled: true, mode: "round_robin", thresholdPercent: 90, members: [members[1]!] } }, "alice");
|
||||
await expect(service().save(pluginKey, { companyId, id: pool.id, expectedRevision: pool.revision, config: saved }, "alice")).rejects.toThrow();
|
||||
expect((await resolve(pool.id, "existing")).binding).toEqual(old.binding);
|
||||
expect((await resolve(pool.id, "future")).memberId).toBe(members[1]!.id);
|
||||
});
|
||||
it("keeps account and harness pinned through independent composer fallback", async () => {
|
||||
const pool = await makePool();
|
||||
const first = await resolve(pool.id, "composer");
|
||||
const changed = await resolve(pool.id, "composer", { overrides: { model: "claude-sonnet-5", modelReasoningEffort: "high" } });
|
||||
expect(changed.binding).toEqual(first.binding); expect(changed.runtimeConfig).toMatchObject({ provider: "codex", model: "gpt-5.6-sol", modelReasoningEffort: "high" }); expect(changed.notes).toHaveLength(1);
|
||||
const claude = poolMemberRuntimeConfig(members[1]!, "paperclip_runner", { model: "gpt-5.6-sol", modelReasoningEffort: "ultra" });
|
||||
expect(claude.config).toMatchObject({ provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-5" }); expect(claude.notes).toHaveLength(2);
|
||||
const overrides = applyAiConnectionRouterTaskSettings({ model: "claude-sonnet-5", effort: "invalid", cwd: "/fixture" }, changed);
|
||||
const native = resolvePaperclipRunnerNativeProviderInput({ backend: "codex_app_server", adapterConfig: projectPaperclipRunnerTaskConfig("codex_app_server", changed.runtimeConfig, overrides) });
|
||||
expect(native).toMatchObject({ provider: "codex", model: "gpt-5.6-sol", codexReasoningEffort: "high" });
|
||||
expect(overrides).toMatchObject({ cwd: "/fixture" }); expect(overrides).not.toHaveProperty("effort");
|
||||
});
|
||||
it("retries a changed config proposal without advancing the abandoned selection", async () => {
|
||||
const pool = await makePool();
|
||||
proposal.mockImplementationOnce(async () => {
|
||||
await service().save(pluginKey, { companyId, id: pool.id, expectedRevision: pool.revision, config: { name: pool.name, enabled: true, mode: pool.mode, thresholdPercent: pool.thresholdPercent, members: [members[1]!] } }, "alice");
|
||||
return { kind: "selected", memberId: members[0]!.id };
|
||||
});
|
||||
expect((await resolve(pool.id, "revision-conflict")).memberId).toBe(members[1]!.id);
|
||||
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(eq(aiConnectionRouterCursors.poolId, pool.id)); expect(cursor?.version).toBe(1);
|
||||
});
|
||||
it("fails closed when the experimental flag changes during a proposal", async () => {
|
||||
const pool = await makePool();
|
||||
proposal.mockImplementationOnce(async () => { await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: false }); return { kind: "selected", memberId: members[0]!.id }; });
|
||||
try { await expect(resolve(pool.id, "disabled-in-flight")).rejects.toThrow("disabled during allocation"); expect(await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, pool.id))).toHaveLength(0); }
|
||||
finally { await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true }); }
|
||||
});
|
||||
it("uses only compatible legacy members and makes reassignment an independent pin", async () => {
|
||||
const pool = await makePool();
|
||||
expect((await resolve(pool.id, "legacy", { adapterType: "claude_local" })).memberId).toBe(members[1]!.id);
|
||||
expect((await resolve(pool.id, "legacy", { agentId: secondAgentId, adapterType: "codex_local" })).memberId).toBe(members[0]!.id);
|
||||
const pins = await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, pool.id)); expect(pins).toHaveLength(2);
|
||||
});
|
||||
it("adopts a valid existing account and requires explicit reset for an unrelated session", async () => {
|
||||
const pool = await makePool();
|
||||
expect((await resolve(pool.id, "adopt", { requireExisting: true, existingGrantId: members[1]!.binding.grantId })).memberId).toBe(members[1]!.id);
|
||||
await expect(resolve(pool.id, "foreign-session", { requireExisting: true, existingGrantId: randomUUID() })).rejects.toThrow("Reset");
|
||||
expect((await resolve(pool.id, "foreign-session")).memberId).toBe(members[0]!.id);
|
||||
});
|
||||
it("rejects cross-company access and unauthorized plugin proposals", async () => {
|
||||
const pool = await makePool();
|
||||
await expect(resolve(pool.id, "foreign", { companyId: otherCompanyId })).rejects.toThrow();
|
||||
proposal.mockResolvedValueOnce({ kind: "selected", memberId: randomUUID() });
|
||||
await expect(resolve(pool.id, "bad-proposal")).rejects.toThrow("unauthorized");
|
||||
expect(await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, pool.id))).toHaveLength(0);
|
||||
});
|
||||
it("does not grant members to another responsible user or move revoked pins", async () => {
|
||||
const pool = await makePool(); const selected = await resolve(pool.id, "revoke");
|
||||
await expect(resolve(pool.id, "revoke", { userId: "bob" })).rejects.toThrow();
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, selected.binding.mode === "responsible_user" ? "" : selected.binding.grantId));
|
||||
try { await expect(resolve(pool.id, "revoke")).rejects.toThrow(); expect((await resolve(pool.id, "new")).memberId).toBe(members[1]!.id); }
|
||||
finally { await db.update(connectionGrants).set({ status: "active" }).where(eq(connectionGrants.id, members[0]!.binding.grantId)); }
|
||||
});
|
||||
it("never probes pure rotation, caches authorized observations, and invalidates on credential change", async () => {
|
||||
const spy = vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response(JSON.stringify({ five_hour: { utilization: 12 } })));
|
||||
try {
|
||||
const pure = await makePool(); await resolve(pure.id, "pure"); expect(spy).not.toHaveBeenCalled();
|
||||
const aware = await makePool({ mode: "usage_aware", members: [members[1]!] }); await resolve(aware.id, "aware"); expect(spy).toHaveBeenCalledTimes(1); await resolve(aware.id, "aware"); expect(spy).toHaveBeenCalledTimes(1);
|
||||
expect((await service().inspect(companyId, aware.id, "alice"))[members[1]!.id]?.checkedAt).not.toBeNull();
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, members[1]!.binding.grantId)); const ref = grant!.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
await secretService(db).rotate(ref.secretId, { value: "fixture-refreshed", preserveAiSessionEpoch: true }, { userId: "alice" });
|
||||
expect((await service().inspect(companyId, aware.id, "alice"))[members[1]!.id]?.checkedAt).toBeNull();
|
||||
await resolve(aware.id, "aware"); expect(spy).toHaveBeenCalledTimes(2);
|
||||
proposal.mockResolvedValueOnce({ kind: "exhausted", retryAt: new Date(Date.now() + 60_000).toISOString(), skipped: {} });
|
||||
await expect(resolve(aware.id, "aware")).rejects.toBeInstanceOf(AiConnectionPoolExhausted);
|
||||
const pins = await db.select().from(aiConnectionTaskPins).where(eq(aiConnectionTaskPins.poolId, aware.id)); expect(pins).toHaveLength(1);
|
||||
} finally { spy.mockRestore(); }
|
||||
});
|
||||
it("expires a usage observation at a reported reset", async () => {
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, members[1]!.binding.grantId));
|
||||
const ref = grant!.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
await secretService(db).rotate(ref.secretId, { value: "fixture-reset-cache", preserveAiSessionEpoch: true }, { userId: "alice" });
|
||||
const spy = vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response(JSON.stringify({ five_hour: { utilization: 12, resets_at: new Date(Date.now() + 200).toISOString() } })));
|
||||
try {
|
||||
const pool = await makePool({ mode: "usage_aware", members: [members[1]!] }); await resolve(pool.id, "reset-cache"); expect(spy).toHaveBeenCalledTimes(1);
|
||||
await new Promise(resolve => setTimeout(resolve, 250)); await resolve(pool.id, "reset-cache"); expect(spy).toHaveBeenCalledTimes(2);
|
||||
} finally { spy.mockRestore(); }
|
||||
});
|
||||
it("treats unfinished probes as unknown within the shared selection budget", async () => {
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, members[1]!.binding.grantId));
|
||||
const ref = grant!.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
await secretService(db).rotate(ref.secretId, { value: "fixture-slow-probe", preserveAiSessionEpoch: true }, { userId: "alice" });
|
||||
const spy = vi.spyOn(globalThis, "fetch").mockImplementation(() => new Promise<Response>(() => {}));
|
||||
try { const pool = await makePool({ mode: "usage_aware", members: [members[1]!] }); const start = Date.now(); expect((await resolve(pool.id, "slow-probe")).memberId).toBe(members[1]!.id); expect(Date.now() - start).toBeLessThan(20_000); expect(proposal.mock.calls.at(-1)?.[2].candidates[0]?.usage).toBeUndefined(); }
|
||||
finally { spy.mockRestore(); }
|
||||
}, 25_000);
|
||||
it("rejects disabled routing but recovers server-owned native evidence after disable/uninstall", async () => {
|
||||
const pool = await makePool(); const selected = await resolve(pool.id, "recovery");
|
||||
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: false });
|
||||
try { await expect(resolve(pool.id, "recovery")).rejects.toThrow("Experimental"); expect(await resolve(pool.id, "recovery", { persisted: selected })).toEqual(selected); }
|
||||
finally { await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true }); }
|
||||
await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, pluginId));
|
||||
try { await expect(resolve(pool.id, "new")).rejects.toThrow("plugin"); expect(await resolve(pool.id, "recovery", { persisted: selected })).toEqual(selected); await db.delete(toolConnections).where(eq(toolConnections.id, pool.id)); expect(await resolve(pool.id, "recovery", { persisted: selected })).toEqual(selected); }
|
||||
finally { await db.update(plugins).set({ status: "ready" }).where(eq(plugins.id, pluginId)); }
|
||||
});
|
||||
it("keeps session identity and fingerprints stable on authenticated refresh, but changes them on manual replacement", async () => {
|
||||
const input = { companyId, agentId, responsibleUserId: "alice", adapterType: "paperclip_runner", binding: members[0]!.binding, config: { provider: "codex", model: "gpt-5.6-sol" } };
|
||||
const first = await prepareManagedAiRuntime(db, input);
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, members[0]!.binding.grantId)); const ref = grant!.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
try {
|
||||
await secretService(db).rotate(ref.secretId, { value: "fixture-new-token", preserveAiSessionEpoch: true }, { userId: "alice" });
|
||||
const refreshed = await prepareManagedAiRuntime(db, input);
|
||||
try { expect(refreshed.identity).not.toBe(first.identity); expect(refreshed.sessionIdentity).toBe(first.sessionIdentity); expect(managedAiSessionFingerprintConfig(refreshed.config, refreshed.home)).toEqual(managedAiSessionFingerprintConfig(first.config, first.home)); } finally { await refreshed.cleanup(); }
|
||||
await secretService(db).rotate(ref.secretId, { value: "fixture-replacement" }, { userId: "alice" });
|
||||
const replaced = await prepareManagedAiRuntime(db, input); try { expect(replaced.sessionIdentity).not.toBe(first.sessionIdentity); } finally { await replaced.cleanup(); }
|
||||
const [secret] = await db.select().from(companySecrets).where(and(eq(companySecrets.companyId, companyId), eq(companySecrets.id, ref.secretId))); expect(secret!.aiSessionEpoch).toBeGreaterThan(0);
|
||||
} finally { await first.cleanup(); }
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,7 @@ describe("instance settings service", () => {
|
||||
expect(normalizeExperimentalSettings({
|
||||
enableEnvironments: true,
|
||||
enableNativeRunner: false,
|
||||
enableAiConnectionRouters: false,
|
||||
enableManagedSandboxOnly: false,
|
||||
enableIsolatedWorkspaces: true,
|
||||
enableIssuePlanDecompositions: true,
|
||||
@@ -35,6 +36,7 @@ describe("instance settings service", () => {
|
||||
})).toEqual({
|
||||
enableEnvironments: true,
|
||||
enableNativeRunner: false,
|
||||
enableAiConnectionRouters: false,
|
||||
enableManagedSandboxOnly: false,
|
||||
enableIsolatedWorkspaces: true,
|
||||
enableIsolatedWorkspacesByDefault: false,
|
||||
|
||||
+1
-1
@@ -897,7 +897,7 @@ export async function createApp(
|
||||
lifecycleManager: lifecycle,
|
||||
instanceInfo: {
|
||||
instanceId: opts.instanceId ?? "default",
|
||||
hostVersion: opts.hostVersion ?? "0.0.0",
|
||||
hostVersion: opts.hostVersion ?? serverVersion,
|
||||
deploymentMode: opts.deploymentMode,
|
||||
deploymentExposure: opts.deploymentExposure,
|
||||
},
|
||||
|
||||
@@ -127,7 +127,7 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
function classifyRetryReasonKind(retryReason: string | null): RetryReasonKind {
|
||||
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) return "max_turn_continuation";
|
||||
if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) return "disposition_repair";
|
||||
if (retryReason === "ai_connection_busy") return "ai_connection_wait";
|
||||
if ((retryReason === "ai_connection_busy" || retryReason === "ai_connection_pool_wait")) return "ai_connection_wait";
|
||||
if (retryReason === "native_safe_replacement") return "native_safe_replacement";
|
||||
return "other";
|
||||
}
|
||||
|
||||
@@ -91,3 +91,8 @@ describe("wake context", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves non-assignee pool wait authority", () => {
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("ai_connection_pool_wait", { aiConnectionBusyDeferredWhileAssignee: false })).toBe(true);
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("ai_connection_pool_wait", { aiConnectionBusyDeferredWhileAssignee: true })).toBe(false);
|
||||
});
|
||||
@@ -17,6 +17,7 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
|
||||
export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation";
|
||||
export const WORKSPACE_BUSY_RETRY_REASON = "workspace_busy";
|
||||
export const AI_CONNECTION_POOL_WAIT_RETRY_REASON = "ai_connection_pool_wait";
|
||||
export const AI_CONNECTION_BUSY_RETRY_REASON = "ai_connection_busy";
|
||||
export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry";
|
||||
export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry";
|
||||
@@ -41,7 +42,7 @@ export function isNonAssigneeWorkspaceBusyRetry(
|
||||
return (
|
||||
(retryReason === WORKSPACE_BUSY_RETRY_REASON &&
|
||||
contextSnapshot.workspaceBusyDeferredWhileAssignee === false) ||
|
||||
(retryReason === AI_CONNECTION_BUSY_RETRY_REASON &&
|
||||
((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) &&
|
||||
contextSnapshot.aiConnectionBusyDeferredWhileAssignee === false)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ export {
|
||||
MAX_TURN_CONTINUATION_RETRY_REASON,
|
||||
WORKSPACE_BUSY_RETRY_REASON,
|
||||
AI_CONNECTION_BUSY_RETRY_REASON,
|
||||
AI_CONNECTION_POOL_WAIT_RETRY_REASON,
|
||||
INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
|
||||
INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
|
||||
WAKE_COMMENT_IDS_KEY,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { aiConnectionRouterService, poolMemberRuntimeConfig } from "../services/ai-connection-router.js";
|
||||
import { connectionIntentService } from "../services/connection-intents.js";
|
||||
import { completeConnectionIntentSchema } from "@paperclipai/shared";
|
||||
import { agentFileStore, agentFileTokenFromHash } from "../services/agent-file-store.js";
|
||||
@@ -5,7 +6,7 @@ import { pipeline } from "node:stream/promises";
|
||||
import { resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared";
|
||||
import { listOpenRouterModels } from "../services/openrouter-models.js";
|
||||
import { prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBindings } from "../services/ai-connection-runtime.js";
|
||||
import { ADAPTER_AUTH_MISSING_CHECK_CODE, AI_CONNECTION_CAPABILITIES, aiConnectionBindingSchema, type AiConnectionBinding } from "@paperclipai/shared";
|
||||
import { ADAPTER_AUTH_MISSING_CHECK_CODE, AI_CONNECTION_CAPABILITIES, aiConnectionBindingSchema, type AiConnectionBinding, aiRuntimeConnectionBindingSchema, type AiRuntimeConnectionBinding } from "@paperclipai/shared";
|
||||
import { toolConnections } from "@paperclipai/db";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import { defaultAiConnectionForHire } from "../services/agent-ai-connection-default.js";
|
||||
@@ -3241,6 +3242,21 @@ export function agentRoutes(
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
const type = assertKnownAdapterType(req.params.type as string);
|
||||
const poolId = asNonEmptyString(req.query.poolId);
|
||||
if (poolId) {
|
||||
const pool = (await aiConnectionRouterService(db).selectable(companyId, getActorInfo(req).actorId)).find(pool => pool.id === poolId);
|
||||
if (!pool) throw unprocessable("Enable an accessible AI connection pool");
|
||||
const catalog = new Map<string, { id: string; label: string }>();
|
||||
for (const member of pool.members) {
|
||||
const memberAdapter = member.binding.provider === "openai" ? "codex_local" : member.binding.provider === "anthropic" ? "claude_local" : member.binding.provider === "xai" ? "grok_local" : "opencode_local";
|
||||
const models = [{ id: member.profile.model, label: member.profile.model }, ...await listAdapterModels(memberAdapter)];
|
||||
for (const model of models) {
|
||||
try { poolMemberRuntimeConfig({ ...member, profile: { ...member.profile, model: model.id } }, type); catalog.set(model.id, model); } catch { /* Only qualified profiles belong in the pool catalog. */ }
|
||||
}
|
||||
}
|
||||
res.json([...catalog.values()]);
|
||||
return;
|
||||
}
|
||||
const refresh = typeof req.query.refresh === "string"
|
||||
? ["1", "true", "yes"].includes(req.query.refresh.toLowerCase())
|
||||
: false;
|
||||
@@ -3379,7 +3395,16 @@ export function agentRoutes(
|
||||
return result;
|
||||
}
|
||||
|
||||
async function validateManagedAgentBinding(req: Request, companyId: string, agentId: string, adapterType: string, config: Record<string, unknown>, binding: AiConnectionBinding, environmentId: string | null | undefined, test: boolean, newAgent = false) {
|
||||
async function validateManagedAgentBinding(req: Request, companyId: string, agentId: string, adapterType: string, config: Record<string, unknown>, binding: AiRuntimeConnectionBinding, environmentId: string | null | undefined, test: boolean, newAgent = false) {
|
||||
if (binding.mode === "router") {
|
||||
|
||||
const settings = await import("../services/instance-settings.js");
|
||||
if (!(await settings.instanceSettingsService(db).getExperimental()).enableAiConnectionRouters) throw unprocessable("Enable AI connection routers in Experimental settings");
|
||||
const pool = (await aiConnectionRouterService(db).selectable(companyId, responsibleUserForAiRequest(req) ?? "")).find((p) => p.id === binding.connectionId && p.enabled);
|
||||
if (!pool) throw unprocessable("Enable the selected connection pool");
|
||||
if (!pool.members.some((m) => { try { poolMemberRuntimeConfig(m, adapterType); return true; } catch { return false; } })) throw unprocessable("This pool has no compatible harness");
|
||||
return pool.id;
|
||||
}
|
||||
const userId = responsibleUserForAiRequest(req);
|
||||
const allowUninstalledShared = newAgent && await canInstallSharedAiConnectionForNewAgent(db, req, companyId, binding);
|
||||
const selection = await aiConnectionService(db).select({ companyId, agentId, userId, adapterType, model: config.model, runnerProvider: config.provider, acpxAgent: config.acpxAgent, binding, allowUninstalledPersonal: newAgent, allowUninstalledShared, allowLegacyValidation: test }).catch((error: unknown) => {
|
||||
@@ -3422,7 +3447,8 @@ export function agentRoutes(
|
||||
|
||||
const adapter = requireServerAdapter(type);
|
||||
|
||||
const aiBinding = req.body.aiConnection ? aiConnectionBindingSchema.parse(req.body.aiConnection) : undefined;
|
||||
const requestedBinding = req.body.aiConnection ? aiRuntimeConnectionBindingSchema.parse(req.body.aiConnection) : undefined;
|
||||
const aiBinding = requestedBinding?.mode === "router" ? undefined : requestedBinding;
|
||||
if (aiBinding && req.body.testCredentials && Object.keys(req.body.testCredentials).length) throw unprocessable("A managed connection test cannot override its credentials");
|
||||
const inputAdapterConfig = aiBinding ? { ...req.body.adapterConfig, env: stripAiAuthBindings(req.body.adapterConfig?.env) } : (req.body?.adapterConfig ?? {}) as Record<string, unknown>;
|
||||
const savedAgentId = typeof req.body.agentId === "string" ? req.body.agentId : null;
|
||||
@@ -3434,6 +3460,12 @@ export function agentRoutes(
|
||||
if (savedAgent.companyId !== companyId) throw notFound("Agent not found");
|
||||
await assertCanUpdateAgent(req, savedAgent);
|
||||
}
|
||||
const poolBinding = requestedBinding ?? (savedAgent?.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(savedAgent.runtimeConfig.aiConnection) : undefined);
|
||||
if (poolBinding?.mode === "router") {
|
||||
await validateManagedAgentBinding(req, companyId, savedAgentId ?? "", type, req.body.adapterConfig ?? {}, poolBinding, req.body.environmentId, false);
|
||||
res.json({ adapterType: type, status: "warn", testedAt: new Date().toISOString(), checks: [{ code: "ai_connection_pool_task_test_required", level: "warn", message: "Pool configuration is available. Run a task to verify the selected account and harness; this preview does not allocate a task or contact a provider." }] });
|
||||
return;
|
||||
}
|
||||
const requestedEnvironmentId = await resolveAdapterTestEnvironmentId(
|
||||
companyId,
|
||||
// Omission tests the saved selection. Explicit null tests a prospective
|
||||
@@ -4632,7 +4664,7 @@ export function agentRoutes(
|
||||
|
||||
const requiresApproval = company.requireBoardApprovalForNewAgents;
|
||||
const status = requiresApproval ? "pending_approval" : "idle";
|
||||
const managedHireBinding = normalizedHireInput.runtimeConfig?.aiConnection ? aiConnectionBindingSchema.parse(normalizedHireInput.runtimeConfig.aiConnection) : undefined;
|
||||
const managedHireBinding = normalizedHireInput.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(normalizedHireInput.runtimeConfig.aiConnection) : undefined;
|
||||
const managedHireConnectionId = managedHireBinding ? await validateManagedAgentBinding(req, companyId, hiredAgentId, normalizedHireInput.adapterType, normalizedHireInput.adapterConfig, managedHireBinding, normalizedHireInput.defaultEnvironmentId, false, true) : undefined;
|
||||
const createdAgent = await svc.create(
|
||||
companyId,
|
||||
@@ -4881,7 +4913,7 @@ export function agentRoutes(
|
||||
allowedSandboxProviders: allowedSandboxProvidersForAgent(createInput.adapterType),
|
||||
});
|
||||
|
||||
const managedBinding = normalizedRuntimeConfig.aiConnection ? aiConnectionBindingSchema.parse(normalizedRuntimeConfig.aiConnection) : undefined;
|
||||
const managedBinding = normalizedRuntimeConfig.aiConnection ? aiRuntimeConnectionBindingSchema.parse(normalizedRuntimeConfig.aiConnection) : undefined;
|
||||
const managedConnectionId = managedBinding ? await validateManagedAgentBinding(req, companyId, agentId, createInput.adapterType, normalizedAdapterConfig, managedBinding, createInput.defaultEnvironmentId, false, true) : undefined;
|
||||
const createdAgent = await svc.create(
|
||||
companyId,
|
||||
@@ -5560,12 +5592,12 @@ export function agentRoutes(
|
||||
});
|
||||
}
|
||||
if (existing.runtimeConfig.aiConnection && requestedRuntimeConfig && !requestedRuntimeConfig.aiConnection) requestedRuntimeConfig.aiConnection = existing.runtimeConfig.aiConnection;
|
||||
const nextAiBinding = aiConnectionBindingSchema.safeParse(requestedRuntimeConfig?.aiConnection ?? existing.runtimeConfig.aiConnection).data;
|
||||
const nextAiBinding = aiRuntimeConnectionBindingSchema.safeParse(requestedRuntimeConfig?.aiConnection ?? existing.runtimeConfig.aiConnection).data;
|
||||
if (nextAiBinding) {
|
||||
await assertCanUpdateAgent(req, existing);
|
||||
const changed = JSON.stringify(nextAiBinding) !== JSON.stringify(existing.runtimeConfig.aiConnection);
|
||||
const aiConfig = (patchData.adapterConfig ?? existing.adapterConfig) as Record<string, unknown>;
|
||||
if (!isAiConnectionCompatible(nextAiBinding, requestedAdapterType, aiConfig.model, aiConfig.provider, aiConfig.acpxAgent)) throw unprocessable("Select an AI connection compatible with the new harness and model");
|
||||
if (nextAiBinding.mode !== "router" && !isAiConnectionCompatible(nextAiBinding, requestedAdapterType, aiConfig.model, aiConfig.provider, aiConfig.acpxAgent)) throw unprocessable("Select an AI connection compatible with the new harness and model");
|
||||
if (changed) await validateManagedAgentBinding(req, existing.companyId, existing.id, requestedAdapterType, aiConfig, nextAiBinding, (patchData.defaultEnvironmentId !== undefined ? patchData.defaultEnvironmentId : existing.defaultEnvironmentId) as string | null, true);
|
||||
}
|
||||
if (requestedRuntimeConfig) patchData.runtimeConfig = requestedRuntimeConfig;
|
||||
|
||||
@@ -22,11 +22,13 @@ import {
|
||||
type AiProvider,
|
||||
type AiConnectionBinding,
|
||||
type AiConnectionList,
|
||||
aiConnectionPoolConfigSchema,
|
||||
} from "@paperclipai/shared";
|
||||
import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||
import { forbidden, notFound, unprocessable } from "../errors.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { logActivity } from "../services/activity-log.js";
|
||||
import { aiConnectionRouterService } from "../services/ai-connection-router.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
|
||||
@@ -173,6 +175,28 @@ export function aiConnectionRoutes(db: Db, options: Parameters<typeof supportsLo
|
||||
const router = Router();
|
||||
const service = aiConnectionService(db);
|
||||
const localLogin = localAiLoginService(db);
|
||||
const pools = aiConnectionRouterService(db);
|
||||
router.get("/companies/:companyId/ai-connection-pools", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoard(req); assertCompanyAccess(req, companyId);
|
||||
if (!(await canManageAiConnections(db, req, companyId))) throw forbidden("Manage connections permission is required");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.json(await pools.list(companyId));
|
||||
});
|
||||
router.get("/companies/:companyId/ai-connection-pools/:poolId/inspection", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoard(req); assertCompanyAccess(req, companyId);
|
||||
if (!(await canManageAiConnections(db, req, companyId))) throw forbidden("Manage connections permission is required");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.json(await pools.inspect(companyId, z.string().uuid().parse(req.params.poolId), getActorInfo(req).actorId));
|
||||
});
|
||||
router.post("/companies/:companyId/ai-connection-pools", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoard(req); assertCompanyAccess(req, companyId);
|
||||
if (!(await canManageAiConnections(db, req, companyId))) throw forbidden("Manage connections permission is required");
|
||||
const body = z.object({ pluginKey: z.string().min(1).max(160), id: z.string().uuid().optional(), expectedRevision: z.number().int().positive().optional(), config: aiConnectionPoolConfigSchema }).strict().parse(req.body);
|
||||
res.json(await pools.save(body.pluginKey, { ...body, companyId }, getActorInfo(req).actorId));
|
||||
});
|
||||
function assertLocalOperator(req: Request) {
|
||||
assertBoard(req);
|
||||
assertCompanyAccess(req, req.params.companyId as string);
|
||||
@@ -216,6 +240,7 @@ export function aiConnectionRoutes(db: Db, options: Parameters<typeof supportsLo
|
||||
throw unprocessable("Invalid agent ID");
|
||||
res.json({
|
||||
currentUserId,
|
||||
pools: await pools.selectable(companyId, currentUserId),
|
||||
canManageConnections: await canManageAiConnections(db, req, companyId),
|
||||
connections: await service.list(
|
||||
companyId,
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { aiConnectionPools, aiConnectionRouterCursors, aiConnectionTaskPins, companySecrets, connectionGrants, instanceSettings, plugins, toolApplications, toolConnections, toolConnectionInstalls, type Db } from "@paperclipai/db";
|
||||
import { aiConnectionPoolConfigSchema, isAiConnectionCompatible, type AiConnectionPool, type AiConnectionPoolMember, type AiConnectionRouterRequest, type AiConnectionRouterSelection, type AiConnectionPoolSaveInput, type AiConnectionUsage } from "@paperclipai/shared";
|
||||
import { isCodexLocalKnownModel, codexLocalReasoningEffortsForModel } from "@paperclipai/adapter-codex-local";
|
||||
import { models as claudeModels, claudeLocalReasoningEffortsForModel } from "@paperclipai/adapter-claude-local";
|
||||
import { models as grokModels, grokLocalReasoningEffortsForModel } from "@paperclipai/adapter-grok-local";
|
||||
import { aiConnectionService } from "./ai-connections.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { forbidden, notFound, conflict, unprocessable } from "../errors.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
import { resolvePaperclipRunnerProviderProfile, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js";
|
||||
|
||||
const usageCache = new Map<string, { expires: number; value: AiConnectionUsage }>();
|
||||
const routedRuntimeKeys = new Set(["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"]);
|
||||
export function applyAiConnectionRouterTaskSettings(config: Record<string, unknown>, selection: AiConnectionRouterSelection) {
|
||||
return { ...Object.fromEntries(Object.entries(config).filter(([key]) => !routedRuntimeKeys.has(key))), ...selection.runtimeConfig };
|
||||
}
|
||||
export class AiConnectionPoolExhausted extends Error {
|
||||
readonly code = "ai_connection_pool_exhausted";
|
||||
constructor(readonly retryAt: string, readonly poolId: string) { super("All eligible pool accounts are over their usage limit. Waiting for usage to reset."); }
|
||||
}
|
||||
|
||||
/** Routing can override provider/model only, never environment, permissions, or instructions. */
|
||||
export function poolMemberRuntimeConfig(member: AiConnectionPoolMember, adapterType: string, overrides: Record<string, unknown> = {}) {
|
||||
const notes: string[] = [];
|
||||
const provider = member.binding.provider;
|
||||
const profile = member.profile;
|
||||
const expected = provider === "openai" ? profile.provider === "codex"
|
||||
: provider === "anthropic" ? profile.provider === "acpx" && profile.acpxAgent === "claude"
|
||||
: provider === "xai" ? profile.provider === "acpx" && profile.acpxAgent === "grok"
|
||||
: profile.provider === "opencode";
|
||||
if (!expected) throw unprocessable("Pool member harness does not match its account");
|
||||
const supportsModel = (value: string) => provider === "openai" ? isCodexLocalKnownModel(value)
|
||||
: provider === "anthropic" ? claudeModels.some((m) => m.id === value)
|
||||
: provider === "xai" ? grokModels.some((m) => m.id === value) : value.startsWith("openrouter/");
|
||||
const config: Record<string, unknown> = { provider: profile.provider, ...(profile.acpxAgent ? { acpxAgent: profile.acpxAgent } : {}), model: profile.model };
|
||||
const validate = () => {
|
||||
if (!isAiConnectionCompatible(member.binding, adapterType, config.model, config.provider, config.acpxAgent)) throw unprocessable("Pool member is incompatible with this harness");
|
||||
if (adapterType === "paperclip_runner") {
|
||||
const resolved = resolvePaperclipRunnerProviderProfile(config);
|
||||
resolvePaperclipRunnerNativeProviderInput({ backend: resolved.backend, adapterConfig: config });
|
||||
}
|
||||
};
|
||||
validate();
|
||||
if (typeof overrides.model === "string" && overrides.model !== profile.model) {
|
||||
if (supportsModel(overrides.model)) {
|
||||
config.model = overrides.model;
|
||||
try { validate(); } catch { config.model = profile.model; notes.push("Model override is unavailable for this member; using its default."); }
|
||||
} else notes.push("Model override is unavailable for this member; using its default.");
|
||||
}
|
||||
const model = String(config.model);
|
||||
const efforts = provider === "openai" ? codexLocalReasoningEffortsForModel(model)
|
||||
: provider === "anthropic" ? claudeLocalReasoningEffortsForModel(model)
|
||||
: provider === "xai" ? grokLocalReasoningEffortsForModel(model) : [];
|
||||
// ACPX currently has no qualified effort override; keep that host limitation explicit.
|
||||
const effortKey = adapterType === "paperclip_runner" ? profile.provider === "codex" ? "modelReasoningEffort" : null
|
||||
: provider === "anthropic" ? "effort" : provider === "xai" ? "reasoningEffort" : provider === "openai" ? "modelReasoningEffort" : "variant";
|
||||
const requested = overrides.modelReasoningEffort ?? overrides.reasoningEffort ?? overrides.effort ?? overrides.variant;
|
||||
const effort = typeof requested === "string" && efforts.includes(requested) && effortKey ? requested
|
||||
: profile.effort && efforts.includes(profile.effort) && effortKey ? profile.effort : null;
|
||||
if (requested && requested !== effort) notes.push("Effort override is unavailable for this member; using its default.");
|
||||
if (effort && effortKey) config[effortKey] = effort;
|
||||
validate();
|
||||
return { config, notes };
|
||||
}
|
||||
|
||||
export function aiConnectionRouterService(db: Db, workerManager?: PluginWorkerManager) {
|
||||
async function owner(pluginKey: string) {
|
||||
const [plugin] = await db.select().from(plugins).where(eq(plugins.pluginKey, pluginKey));
|
||||
if (!plugin || plugin.status !== "ready" || !plugin.manifestJson.capabilities.includes("ai.connections.route")) throw unprocessable("Enable a compatible AI connection router plugin");
|
||||
return plugin;
|
||||
}
|
||||
async function enabled() {
|
||||
if (!(await instanceSettingsService(db).getExperimental()).enableAiConnectionRouters) throw unprocessable("Enable AI connection routers in Experimental settings", { code: "ai_connection_router_disabled" });
|
||||
}
|
||||
async function list(companyId: string) {
|
||||
const rows = await db.select().from(aiConnectionPools).where(eq(aiConnectionPools.companyId, companyId));
|
||||
return rows.map((row): AiConnectionPool => ({ ...row.config, id: row.id, companyId, pluginKey: row.pluginKey, revision: row.revision }));
|
||||
}
|
||||
async function selectable(companyId: string, userId: string) {
|
||||
if (!(await instanceSettingsService(db).getExperimental()).enableAiConnectionRouters) return [];
|
||||
const accounts = await aiConnectionService(db).list(companyId, userId);
|
||||
const result: AiConnectionPool[] = [];
|
||||
for (const pool of await list(companyId)) {
|
||||
if (!pool.enabled) continue;
|
||||
try { await owner(pool.pluginKey); } catch { continue; }
|
||||
const members = pool.members.filter(member => accounts.some(account => account.id === member.binding.connectionId && account.grantId === member.binding.grantId));
|
||||
if (members.length) result.push({ ...pool, members });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async function inspect(companyId: string, poolId: string, userId: string) {
|
||||
const pool = (await list(companyId)).find(pool => pool.id === poolId);
|
||||
if (!pool) throw notFound("Connection pool not found");
|
||||
const visible = await aiConnectionService(db).list(companyId, userId);
|
||||
const result: Record<string, { reason: string; checkedAt: string | null; usage?: AiConnectionUsage }> = {};
|
||||
for (const member of pool.members) {
|
||||
const account = visible.find(account => account.id === member.binding.connectionId && account.grantId === member.binding.grantId);
|
||||
const key = account ? await usageCacheKey(member, companyId) : null;
|
||||
const cached = key ? usageCache.get(key) : undefined;
|
||||
const observation = cached && cached.expires > Date.now() ? cached.value : undefined;
|
||||
result[member.id] = { reason: account?.status === "connected" ? "Eligible if agent access permits" : account?.status ?? "Not permitted", checkedAt: account && observation ? observation.checkedAt : null, ...(account && observation ? { usage: observation } : {}) };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async function save(pluginKey: string, input: AiConnectionPoolSaveInput, userId: string) {
|
||||
await enabled();
|
||||
const plugin = await owner(pluginKey);
|
||||
const config = aiConnectionPoolConfigSchema.parse(input.config);
|
||||
const visible = await aiConnectionService(db).list(input.companyId, userId);
|
||||
for (const member of config.members) {
|
||||
if (!visible.some((v) => v.id === member.binding.connectionId && v.grantId === member.binding.grantId)) throw forbidden("Choose an account you may use in this company");
|
||||
poolMemberRuntimeConfig(member, "paperclip_runner");
|
||||
}
|
||||
const id = input.id ?? randomUUID();
|
||||
await db.transaction(async (tx) => {
|
||||
if (input.id) {
|
||||
const [row] = await tx.select().from(aiConnectionPools).where(and(eq(aiConnectionPools.id, id), eq(aiConnectionPools.companyId, input.companyId), eq(aiConnectionPools.pluginKey, pluginKey))).for("update");
|
||||
if (!row) throw notFound("Connection pool not found");
|
||||
if (input.expectedRevision !== row.revision) throw conflict("Pool changed; reload before saving");
|
||||
await tx.update(aiConnectionPools).set({ config, revision: row.revision + 1, updatedAt: new Date() }).where(eq(aiConnectionPools.id, id));
|
||||
await tx.update(toolConnections).set({ name: config.name, enabled: config.enabled, updatedAt: new Date() }).where(and(eq(toolConnections.id, id), eq(toolConnections.companyId, input.companyId)));
|
||||
} else {
|
||||
const key = `plugin:${pluginKey}:ai-router`;
|
||||
await tx.insert(toolApplications).values({ companyId: input.companyId, applicationKey: key, name: `${plugin.manifestJson.displayName} connections`, type: "paperclip_plugin", pluginId: plugin.id }).onConflictDoNothing();
|
||||
const [application] = await tx.select().from(toolApplications).where(and(eq(toolApplications.companyId, input.companyId), eq(toolApplications.applicationKey, key)));
|
||||
if (!application) throw new Error("Router application could not be created");
|
||||
await tx.insert(toolConnections).values({ id, companyId: input.companyId, applicationId: application.id, name: config.name, uid: id, connectionPurpose: "ai", transport: "runtime_auth", enabled: config.enabled, status: "active", healthStatus: "ok", config: { aiRouter: { pluginKey } }, createdByUserId: userId });
|
||||
await tx.insert(aiConnectionPools).values({ id, companyId: input.companyId, pluginKey, config });
|
||||
await tx.insert(aiConnectionRouterCursors).values({ poolId: id, companyId: input.companyId });
|
||||
await tx.insert(toolConnectionInstalls).values({ companyId: input.companyId, connectionId: id, targetType: "company", targetId: input.companyId, createdByUserId: userId });
|
||||
}
|
||||
await logActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "user", actorId: userId, action: "ai_connection.pool_saved", entityType: "tool_connection", entityId: id, details: { enabled: config.enabled, memberCount: config.members.length, mode: config.mode } });
|
||||
});
|
||||
return (await list(input.companyId)).find((p) => p.id === id)!;
|
||||
}
|
||||
async function usageCacheKey(member: AiConnectionPoolMember, companyId: string) {
|
||||
const [grant] = await db.select().from(connectionGrants).where(and(eq(connectionGrants.companyId, companyId), eq(connectionGrants.id, member.binding.grantId)));
|
||||
if (!grant) return null;
|
||||
const ref = grant.credentialSecretRefs.find((r) => r.configPath === "ai.credential");
|
||||
const [secret] = ref ? await db.select({ version: companySecrets.latestVersion }).from(companySecrets).where(and(eq(companySecrets.companyId, companyId), eq(companySecrets.id, ref.secretId))) : [];
|
||||
return `${companyId}:${grant.id}:${secret?.version}:${grant.updatedAt.toISOString()}`;
|
||||
}
|
||||
async function probe(member: AiConnectionPoolMember, companyId: string, userId: string, deadline: number) {
|
||||
const key = await usageCacheKey(member, companyId);
|
||||
if (!key) return undefined;
|
||||
const cached = usageCache.get(key);
|
||||
if (cached && cached.expires > Date.now()) return cached.value;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const value = await Promise.race([
|
||||
aiConnectionService(db).probeUsage(companyId, userId, member.binding.connectionId, member.binding.grantId).catch(() => undefined),
|
||||
new Promise<undefined>((resolve) => { timer = setTimeout(() => resolve(undefined), Math.max(0, deadline - Date.now())); }),
|
||||
]);
|
||||
if (value) {
|
||||
const resets = value.limits.map((v) => Date.parse(v.resetsAt ?? "")).filter((v) => Number.isFinite(v) && v > Date.now());
|
||||
if (usageCache.size > 1000) usageCache.clear();
|
||||
usageCache.set(key, { value, expires: Math.min(Date.now() + 60_000, ...resets) });
|
||||
}
|
||||
return value;
|
||||
} finally { if (timer) clearTimeout(timer); }
|
||||
}
|
||||
async function resolve(input: { companyId: string; poolId: string; agentId: string; userId: string | null; adapterType: string; taskKey: string; overrides?: Record<string, unknown>; existingGrantId?: string; requireExisting?: boolean; persisted?: AiConnectionRouterSelection }) {
|
||||
const pinWhere = and(eq(aiConnectionTaskPins.companyId, input.companyId), eq(aiConnectionTaskPins.poolId, input.poolId), eq(aiConnectionTaskPins.agentId, input.agentId), eq(aiConnectionTaskPins.taskKey, input.taskKey));
|
||||
const validateSelection = async (selection: AiConnectionRouterSelection, client = db) => aiConnectionService(client).select({ companyId: input.companyId, agentId: input.agentId, userId: input.userId, adapterType: input.adapterType, binding: selection.binding, model: selection.runtimeConfig.model, runnerProvider: selection.runtimeConfig.provider, acpxAgent: selection.runtimeConfig.acpxAgent });
|
||||
if (input.persisted) {
|
||||
if (input.persisted.poolId !== input.poolId) throw unprocessable("Persisted pool selection does not match this run");
|
||||
const [pin] = await db.select().from(aiConnectionTaskPins).where(pinWhere);
|
||||
// Uninstall can remove the virtual connection. Server-owned native recovery
|
||||
// evidence still authorizes the original concrete selection, never a new one.
|
||||
if (pin && (pin.selection.memberId !== input.persisted.memberId || JSON.stringify(pin.selection.binding) !== JSON.stringify(input.persisted.binding))) throw unprocessable("Persisted pool allocation is inconsistent");
|
||||
await validateSelection(input.persisted);
|
||||
return input.persisted;
|
||||
}
|
||||
await enabled();
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
const [row] = await db.select().from(aiConnectionPools).where(and(eq(aiConnectionPools.id, input.poolId), eq(aiConnectionPools.companyId, input.companyId)));
|
||||
if (!row || !row.config.enabled) throw unprocessable("Enable the selected connection pool");
|
||||
const plugin = await owner(row.pluginKey);
|
||||
if (!workerManager?.isRunning(plugin.id)) throw unprocessable("The connection router worker is unavailable; retry when it is ready");
|
||||
const [pin] = await db.select().from(aiConnectionTaskPins).where(pinWhere);
|
||||
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(and(eq(aiConnectionRouterCursors.poolId, row.id), eq(aiConnectionRouterCursors.companyId, input.companyId)));
|
||||
if (!cursor) throw unprocessable("The connection pool cursor is unavailable");
|
||||
const pool: AiConnectionPool = { ...row.config, id: row.id, companyId: row.companyId, pluginKey: row.pluginKey, revision: row.revision };
|
||||
const deadline = Date.now() + 15_000;
|
||||
const candidates: AiConnectionRouterRequest["candidates"] = [];
|
||||
for (const member of pin ? [pin.member] : pool.members) {
|
||||
try {
|
||||
const effective = poolMemberRuntimeConfig(member, input.adapterType, input.overrides);
|
||||
const selection: AiConnectionRouterSelection = { poolId: row.id, memberId: member.id, binding: member.binding, runtimeConfig: effective.config, notes: effective.notes };
|
||||
await validateSelection(selection);
|
||||
candidates.push({ member, runtimeConfig: effective.config, notes: effective.notes });
|
||||
} catch (error) { if (pin) throw error; }
|
||||
}
|
||||
if (!pin && input.requireExisting) {
|
||||
const existing = candidates.find((candidate) => candidate.member.binding.grantId === input.existingGrantId);
|
||||
if (!existing) throw unprocessable("Reset this task session before adopting the connection pool");
|
||||
candidates.splice(0, candidates.length, existing);
|
||||
}
|
||||
if (pool.mode === "usage_aware" && input.userId) {
|
||||
// At most four provider calls at once; the shared deadline bounds the whole pool.
|
||||
for (let i = 0; i < candidates.length && Date.now() < deadline; i += 4) await Promise.all(candidates.slice(i, i + 4).map(async (candidate) => { candidate.usage = await probe(candidate.member, input.companyId, input.userId!, deadline); }));
|
||||
}
|
||||
const authorizedPool = { ...pool, members: candidates.map((candidate) => candidate.member) };
|
||||
const result = await workerManager.call(plugin.id, "routeAiConnection", { companyId: input.companyId, pool: authorizedPool, memberOrder: pool.members.map((member) => member.id), agentId: input.agentId, taskKey: input.taskKey, lastMemberId: cursor.lastMemberId, cursorVersion: cursor.version, ...(pin ? { pinnedMemberId: pin.member.id } : {}), candidates, now: new Date().toISOString() }, 5000);
|
||||
if (result.kind === "exhausted") {
|
||||
const retry = Date.parse(result.retryAt);
|
||||
throw new AiConnectionPoolExhausted(new Date(Number.isFinite(retry) ? Math.max(Date.now() + 1000, retry) : Date.now() + 60_000).toISOString(), pool.id);
|
||||
}
|
||||
if (result.kind !== "selected") throw unprocessable("No pool account is permitted and compatible for this task");
|
||||
const candidate = candidates.find((v) => v.member.id === result.memberId);
|
||||
if (!candidate || (pin && candidate.member.id !== pin.member.id)) throw unprocessable("The router returned an unauthorized member");
|
||||
const selected: AiConnectionRouterSelection = { poolId: row.id, memberId: candidate.member.id, binding: candidate.member.binding, runtimeConfig: candidate.runtimeConfig, notes: candidate.notes };
|
||||
if (pin) return selected;
|
||||
const committed = await db.transaction(async (tx) => {
|
||||
const [locked] = await tx.select().from(aiConnectionRouterCursors).where(and(eq(aiConnectionRouterCursors.poolId, row.id), eq(aiConnectionRouterCursors.companyId, input.companyId))).for("update");
|
||||
const [existing] = await tx.select().from(aiConnectionTaskPins).where(pinWhere);
|
||||
if (existing) return { retry: true };
|
||||
const [current] = await tx.select().from(aiConnectionPools).where(and(eq(aiConnectionPools.id, row.id), eq(aiConnectionPools.companyId, input.companyId))).for("share");
|
||||
if (!locked || locked.version !== cursor.version || !current || current.revision !== row.revision) return { retry: true };
|
||||
const [currentPlugin] = await tx.select().from(plugins).where(eq(plugins.id, plugin.id)).for("share");
|
||||
if (currentPlugin?.status !== "ready") throw unprocessable("The router plugin was disabled during allocation");
|
||||
await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(eq(instanceSettings.singletonKey, "default")).for("share");
|
||||
if (!(await instanceSettingsService(tx as unknown as Db).getExperimental()).enableAiConnectionRouters) throw unprocessable("AI connection routers were disabled during allocation");
|
||||
await validateSelection(selected, tx as unknown as Db);
|
||||
await tx.insert(aiConnectionTaskPins).values({ companyId: input.companyId, poolId: row.id, agentId: input.agentId, taskKey: input.taskKey, selection: selected, member: candidate.member });
|
||||
await tx.update(aiConnectionRouterCursors).set({ lastMemberId: candidate.member.id, version: sql`${aiConnectionRouterCursors.version} + 1` }).where(eq(aiConnectionRouterCursors.poolId, row.id));
|
||||
await logActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "system", agentId: input.agentId, action: "ai_connection.pool_selected", entityType: "tool_connection", entityId: row.id, details: { memberId: candidate.member.id, connectionId: candidate.member.binding.connectionId, taskKey: input.taskKey } });
|
||||
return { retry: false };
|
||||
});
|
||||
if (!committed.retry) return selected;
|
||||
}
|
||||
throw conflict("Concurrent pool selections changed the cursor; retry this task");
|
||||
}
|
||||
return { list, selectable, inspect, save, resolve };
|
||||
}
|
||||
@@ -189,7 +189,13 @@ export function managedAiSessionFingerprintConfig(
|
||||
for (const [key, value] of Object.entries(managedAiHomeEnvironment(managedHome))) {
|
||||
if (env[key] === value) env[key] = stable[key];
|
||||
}
|
||||
return { ...config, env };
|
||||
const managed = config.managedAiConnection as Record<string, unknown> | undefined;
|
||||
if (managed?.sessionIdentity) {
|
||||
for (const key of ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN", "OPENAI_API_KEY", "CODEX_API_KEY", "OPENROUTER_API_KEY", "XAI_API_KEY", "GROK_API_KEY", "OPENCODE_AUTH_JSON", "OPENCODE_CONFIG_CONTENT"]) {
|
||||
if (env[key]) env[key] = "<managed-ai-credential>";
|
||||
}
|
||||
}
|
||||
return { ...config, env, ...(managed?.sessionIdentity ? { managedAiConnection: { ...managed, identity: managed.sessionIdentity } } : {}) };
|
||||
}
|
||||
|
||||
export async function prepareManagedAiRuntime(
|
||||
@@ -251,7 +257,14 @@ export async function prepareManagedAiRuntime(
|
||||
throw unprocessable(
|
||||
"The selected default changed. Retry this execution.",
|
||||
);
|
||||
const credentialRef = selection.grant.credentialSecretRefs.find((ref) => ref.configPath === "ai.credential");
|
||||
if (!credentialRef) throw unprocessable("The selected AI credential is unavailable");
|
||||
const readFreshness = async () => (await db.select({ epoch: companySecrets.aiSessionEpoch, version: companySecrets.latestVersion })
|
||||
.from(companySecrets).where(and(eq(companySecrets.companyId, input.companyId), eq(companySecrets.id, credentialRef.secretId))).limit(1))[0];
|
||||
const freshness = await readFreshness();
|
||||
const value = await service.credential(selection);
|
||||
const afterRead = await readFreshness();
|
||||
if (!freshness || freshness.version !== afterRead?.version || freshness.epoch !== afterRead.epoch) throw unprocessable("The AI credential changed during preparation; retry this execution");
|
||||
home = await mkdtemp(
|
||||
path.join(
|
||||
os.tmpdir(),
|
||||
@@ -298,11 +311,13 @@ export async function prepareManagedAiRuntime(
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
const identity = `${selection.grant.id}:${input.responsibleUserId ?? "shared"}:${generation}`;
|
||||
const sessionIdentity = `${selection.grant.id}:${input.responsibleUserId ?? "shared"}:${freshness.epoch}`;
|
||||
return {
|
||||
sessionIdentity,
|
||||
config: {
|
||||
...input.config,
|
||||
env,
|
||||
managedAiConnection: { ...selection.attribution, identity },
|
||||
managedAiConnection: { ...selection.attribution, identity, sessionIdentity },
|
||||
},
|
||||
attribution: selection.attribution,
|
||||
accountName: selection.connection.name,
|
||||
@@ -367,7 +382,7 @@ export async function prepareManagedAiRuntime(
|
||||
if (decision !== 10) return;
|
||||
await secretService(tx).rotate(
|
||||
ref.secretId,
|
||||
{ value: refreshed },
|
||||
{ value: refreshed, preserveAiSessionEpoch: true },
|
||||
{ userId: grant.subjectUserId },
|
||||
);
|
||||
await tx
|
||||
|
||||
@@ -27,7 +27,7 @@ describe("failure attempts across resource waits", () => {
|
||||
});
|
||||
|
||||
describe("persisted independent accounting", () => {
|
||||
it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => {
|
||||
it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => {
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason, scheduledRetryAttempt: 20,
|
||||
contextSnapshot: { executionRetryAccounting: { version: 1, failureRetries: 2, maxTurnContinuations: 1 } },
|
||||
})).toBe(2);
|
||||
|
||||
@@ -28,7 +28,7 @@ function savedAccounting(run: RetryRun): ExecutionRetryAccounting | null {
|
||||
|
||||
function historicalFailureCount(run: RetryRun): number {
|
||||
if (run.scheduledRetryReason === "max_turns_continuation" || run.scheduledRetryReason === "issue_disposition_repair") return 0;
|
||||
if (run.scheduledRetryReason === "ai_connection_busy") {
|
||||
if (run.scheduledRetryReason === "ai_connection_busy" || run.scheduledRetryReason === "ai_connection_pool_wait") {
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeAiConnectionWait);
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
@@ -42,7 +42,7 @@ function historicalFailureCount(run: RetryRun): number {
|
||||
|
||||
export function executionRetryAccounting(run: RetryRun): ExecutionRetryAccounting {
|
||||
const saved = savedAccounting(run);
|
||||
const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"].includes(run.scheduledRetryReason ?? "");
|
||||
const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"].includes(run.scheduledRetryReason ?? "");
|
||||
return {
|
||||
version: 1,
|
||||
failureRetries: Math.max(saved?.failureRetries ?? 0, saved && nonFailureLane ? 0 : historicalFailureCount(run)),
|
||||
@@ -57,7 +57,7 @@ export function executionFailureRetryCount(run: RetryRun): number {
|
||||
}
|
||||
|
||||
export function executionRetryAttemptCount(run: RetryRun, reason: string): number {
|
||||
if (reason === "workspace_busy" || reason === "ai_connection_busy") {
|
||||
if (reason === "workspace_busy" || reason === "ai_connection_busy" || reason === "ai_connection_pool_wait") {
|
||||
return run.scheduledRetryReason === reason ? count(run.scheduledRetryAttempt) ?? 0 : 0;
|
||||
}
|
||||
const accounting = executionRetryAccounting(run);
|
||||
@@ -67,6 +67,6 @@ export function executionRetryAttemptCount(run: RetryRun, reason: string): numbe
|
||||
export function accountingForScheduledRetry(run: RetryRun, reason: string, attempt: number): ExecutionRetryAccounting {
|
||||
const accounting = executionRetryAccounting(run);
|
||||
if (reason === "max_turns_continuation") accounting.maxTurnContinuations = attempt;
|
||||
else if (reason !== "workspace_busy" && reason !== "ai_connection_busy") accounting.failureRetries = attempt;
|
||||
else if (reason !== "workspace_busy" && reason !== "ai_connection_busy" && reason !== "ai_connection_pool_wait") accounting.failureRetries = attempt;
|
||||
return accounting;
|
||||
}
|
||||
@@ -34,7 +34,8 @@ import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAs
|
||||
import { admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js";
|
||||
import { connectionIntentService } from "./connection-intents.js";
|
||||
import { managedAiSessionFingerprintConfig, prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBindings, isAiConnectionBusy, AI_AUTH_ENV_KEYS } from "./ai-connection-runtime.js";
|
||||
import { aiConnectionBindingSchema } from "@paperclipai/shared";
|
||||
import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared";
|
||||
import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js";
|
||||
import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js";
|
||||
import { CONVERSATION_CONTINUATION_POLICY, claimedAdapterType, runUsedConversationAdapter, hasConversationContinuationPolicy, isConversationAdapter } from "./conversation-continuation.js";
|
||||
import { recordExecutionWait } from "./execution-wait.js";
|
||||
@@ -535,6 +536,7 @@ import {
|
||||
MAX_TURN_CONTINUATION_RETRY_REASON,
|
||||
WORKSPACE_BUSY_RETRY_REASON,
|
||||
AI_CONNECTION_BUSY_RETRY_REASON,
|
||||
AI_CONNECTION_POOL_WAIT_RETRY_REASON,
|
||||
INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
|
||||
INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
|
||||
WAKE_COMMENT_IDS_KEY,
|
||||
@@ -15542,7 +15544,7 @@ export function heartbeatService(
|
||||
const requiresIssueGate =
|
||||
isTransientWorkspaceGitScanCode(run.errorCode) ||
|
||||
hasConversationContinuationPolicy(run.resultJson) ||
|
||||
retryReason === AI_CONNECTION_BUSY_RETRY_REASON ||
|
||||
(retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) ||
|
||||
retryReason === MAX_TURN_CONTINUATION_RETRY_REASON ||
|
||||
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON;
|
||||
if (requiresIssueGate) {
|
||||
@@ -15612,7 +15614,7 @@ export function heartbeatService(
|
||||
executionFailureRetryCount(run),
|
||||
}
|
||||
: {}),
|
||||
...(retryReason === AI_CONNECTION_BUSY_RETRY_REASON
|
||||
...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON)
|
||||
? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) }
|
||||
: {}),
|
||||
...(shouldQuarantineWorkspaceForRetry
|
||||
@@ -15938,7 +15940,7 @@ export function heartbeatService(
|
||||
}
|
||||
|
||||
if (
|
||||
retryReason === AI_CONNECTION_BUSY_RETRY_REASON && issueId &&
|
||||
(retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId &&
|
||||
!isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot)
|
||||
) {
|
||||
// The issue row is locked above. Recheck after the preflight gate so
|
||||
@@ -20433,7 +20435,7 @@ export function heartbeatService(
|
||||
let readFailureReportSecrets: () => string[] = () => [];
|
||||
|
||||
try {
|
||||
const agent = await getAgent(run.agentId);
|
||||
let agent = await getAgent(run.agentId);
|
||||
if (!agent) {
|
||||
await setRunStatus(runId, "failed", {
|
||||
error: "Agent not found",
|
||||
@@ -20684,7 +20686,7 @@ export function heartbeatService(
|
||||
: row;
|
||||
})
|
||||
: null;
|
||||
const issueAssigneeOverrides =
|
||||
let issueAssigneeOverrides =
|
||||
issueContext && issueContext.assigneeAgentId === agent.id
|
||||
? parseIssueAssigneeAdapterOverrides(
|
||||
issueContext.assigneeAdapterOverrides,
|
||||
@@ -20890,7 +20892,7 @@ export function heartbeatService(
|
||||
// durable for authorization and operation-time credential resolution.
|
||||
context.executionPolicy = retainedTrust.executionPolicy;
|
||||
}
|
||||
const config = parseObject(agent.adapterConfig);
|
||||
let config = parseObject(agent.adapterConfig);
|
||||
const taskSession = taskKey
|
||||
? await getTaskSession(
|
||||
agent.companyId,
|
||||
@@ -20899,6 +20901,46 @@ export function heartbeatService(
|
||||
taskKey,
|
||||
)
|
||||
: null;
|
||||
const routerHasPersistedInput = Object.keys(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput)).length > 0;
|
||||
const persistedRouterPoolId = routerHasPersistedInput ? readNonEmptyString(parseObject(context.aiRouterSelection).poolId) : null;
|
||||
const requestedAiBinding = persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
|
||||
let aiBinding = requestedAiBinding?.mode === "router" ? undefined : requestedAiBinding;
|
||||
if (requestedAiBinding?.mode === "router") {
|
||||
try {
|
||||
context.aiRouterTaskKey = readNonEmptyString(context.aiRouterTaskKey) ?? taskKey ?? run.id;
|
||||
const savedIdentity = taskSession?.sessionParamsJson?.paperclipAiCredentialIdentity;
|
||||
const selection = await aiConnectionRouterService(db, options.pluginWorkerManager).resolve({
|
||||
companyId: agent.companyId, poolId: requestedAiBinding.connectionId, agentId: agent.id,
|
||||
userId: responsibleUserId, adapterType: agent.adapterType, taskKey: String(context.aiRouterTaskKey),
|
||||
overrides: issueAssigneeOverrides?.adapterConfig ?? {},
|
||||
existingGrantId: typeof savedIdentity === "string" ? savedIdentity.split(":")[0] : undefined,
|
||||
requireExisting: Boolean(taskSession?.sessionDisplayId) && !shouldResetTaskSessionForWake(context),
|
||||
persisted: routerHasPersistedInput ? context.aiRouterSelection as AiConnectionRouterSelection | undefined : undefined,
|
||||
});
|
||||
aiBinding = selection.binding;
|
||||
// Provider-specific fields from the configured harness must not leak into the selected member.
|
||||
config = applyAiConnectionRouterTaskSettings(config, selection);
|
||||
if (issueAssigneeOverrides) issueAssigneeOverrides = { ...issueAssigneeOverrides, adapterConfig: applyAiConnectionRouterTaskSettings(issueAssigneeOverrides.adapterConfig ?? {}, selection) };
|
||||
agent = { ...agent, adapterConfig: config };
|
||||
context.aiRouterSelection = selection;
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ aiRouterTaskKey: context.aiRouterTaskKey, aiRouterSelection: selection })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
|
||||
await appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "info", message: "Using task-pinned pool account", payload: { poolId: selection.poolId, memberId: selection.memberId, provider: aiBinding.provider, model: config.model, notes: selection.notes } });
|
||||
} catch (error) {
|
||||
if (error instanceof AiConnectionPoolExhausted && !routerHasPersistedInput) {
|
||||
const now = new Date();
|
||||
context.aiConnectionBusyDeferredWhileAssignee = issueContext?.assigneeAgentId === agent.id;
|
||||
const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { error: error.message, errorCode: error.code, finishedAt: now, resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, retryAt: error.retryAt }, contextSnapshot: context });
|
||||
if (cancelled.updated) {
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message });
|
||||
const retry = await scheduleBoundedRetryForRun(cancelled.run ?? run, agent, { now, retryReason: AI_CONNECTION_POOL_WAIT_RETRY_REASON, wakeReason: "ai_connection_pool_retry", maxAttempts: (run.scheduledRetryAttempt ?? 0) + 1, delayMs: Math.max(1000, Date.parse(error.retryAt) - Date.now()) });
|
||||
if (retry.outcome !== "scheduled") await releaseIssueExecutionAndPromote(cancelled.run ?? run);
|
||||
await finalizeAgentStatus(run.agentId, "cancelled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
throw new ConfigurationIncompleteFailure(error instanceof Error ? error.message : "Configure this connection pool", { configurationIncomplete: { reason: "ai_connection_unavailable", companyId: agent.companyId, agentId: agent.id, responsibleUserId, actionUrl: `/agents/${agent.id}/runtime`, fingerprint: `ai-router:${requestedAiBinding.connectionId}` } });
|
||||
}
|
||||
}
|
||||
if (isConversation(issueContext)) {
|
||||
delete context.resumeSessionParams;
|
||||
delete context.resumeSessionDisplayId;
|
||||
@@ -21569,7 +21611,8 @@ export function heartbeatService(
|
||||
});
|
||||
const mergedConfig = {
|
||||
...workspaceManagedConfig,
|
||||
...(issueAssigneeOverrides?.adapterConfig ?? {}),
|
||||
...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))),
|
||||
...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}),
|
||||
// The base below is already task-owned. Keep directory transport while
|
||||
// preserving isolated mode and the mandatory sandbox preflight.
|
||||
...(useIsolatedTaskDirectory ? { workspaceStrategy: { type: "project_primary" } } : {}),
|
||||
@@ -21606,7 +21649,7 @@ export function heartbeatService(
|
||||
["local", "ssh"].includes(
|
||||
selectedEnvironmentForConfig?.driver ?? "local",
|
||||
);
|
||||
const aiBinding = agent.runtimeConfig?.aiConnection ? aiConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
|
||||
|
||||
const { resolvedConfig, secretKeys, secretManifest } =
|
||||
await resolveExecutionRunAdapterConfig({
|
||||
managedAiCredentials: Boolean(aiBinding),
|
||||
@@ -21656,13 +21699,16 @@ export function heartbeatService(
|
||||
fingerprint: `ai:${agent.id}:${responsibleUserId}:${JSON.stringify(aiBinding)}` },
|
||||
});
|
||||
}
|
||||
if (persistedNativeExecutionInput && parseObject(run.contextSnapshot?.aiConnection).identity !== managedAiRuntime.identity) {
|
||||
const savedAiAccount = parseObject(run.contextSnapshot?.aiConnection);
|
||||
if (persistedNativeExecutionInput && (savedAiAccount.sessionIdentity
|
||||
? savedAiAccount.sessionIdentity !== managedAiRuntime.sessionIdentity
|
||||
: savedAiAccount.identity !== managedAiRuntime.identity)) {
|
||||
throw new ConfigurationIncompleteFailure("The AI account changed while this native run was suspended. Start a new execution.", { configurationIncomplete: { reason: "ai_connection_changed", actionUrl: `/agents/${agent.id}/runtime` } });
|
||||
}
|
||||
Object.assign(resolvedConfig, managedAiRuntime.config);
|
||||
for (const key of AI_AUTH_ENV_KEYS) secretKeys.add(key);
|
||||
context.aiConnection = { ...managedAiRuntime.attribution, identity: managedAiRuntime.identity };
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ aiConnection: context.aiConnection })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
|
||||
context.aiConnection = { ...managedAiRuntime.attribution, accountName: managedAiRuntime.accountName, identity: managedAiRuntime.identity, sessionIdentity: managedAiRuntime.sessionIdentity };
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ aiConnection: context.aiConnection, ...(context.aiRouterTaskKey ? { aiRouterTaskKey: context.aiRouterTaskKey } : {}), ...(context.aiRouterSelection ? { aiRouterSelection: context.aiRouterSelection } : {}) })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
if (secretManifest.length > 0) {
|
||||
context.paperclipSecrets = {
|
||||
@@ -23044,10 +23090,10 @@ export function heartbeatService(
|
||||
|
||||
const taskSessionCredentialCompatible = isTaskSessionCredentialCompatible(
|
||||
taskSession?.sessionParamsJson,
|
||||
managedAiRuntime?.identity,
|
||||
);
|
||||
managedAiRuntime?.sessionIdentity,
|
||||
) || Boolean(managedAiRuntime && isTaskSessionCredentialCompatible(taskSession?.sessionParamsJson, managedAiRuntime.identity));
|
||||
if (managedAiRuntime) {
|
||||
sessionConfigMetadata.aiCredentialIdentity = managedAiRuntime.identity;
|
||||
sessionConfigMetadata.aiCredentialIdentity = managedAiRuntime.sessionIdentity;
|
||||
if (!taskSessionCredentialCompatible) {
|
||||
runtimeSessionIdForAdapter = null;
|
||||
runtimeSessionParamsForAdapter = null;
|
||||
|
||||
@@ -223,6 +223,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
return {
|
||||
enableEnvironments: parsed.data.enableEnvironments ?? false,
|
||||
enableNativeRunner: parsed.data.enableNativeRunner ?? true,
|
||||
enableAiConnectionRouters: parsed.data.enableAiConnectionRouters ?? false,
|
||||
enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false,
|
||||
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
|
||||
enableIsolatedWorkspacesByDefault: parsed.data.enableIsolatedWorkspacesByDefault ?? false,
|
||||
@@ -268,6 +269,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
return {
|
||||
enableEnvironments: false,
|
||||
enableNativeRunner: true,
|
||||
enableAiConnectionRouters: false,
|
||||
enableManagedSandboxOnly: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
enableIsolatedWorkspacesByDefault: false,
|
||||
|
||||
@@ -2397,6 +2397,8 @@ export function secretService(db: Db | DbTransaction) {
|
||||
// update matches the latest version, so a concurrent rotation between the
|
||||
// read and the write cannot pass. A mismatch throws a 409 conflict.
|
||||
expectedLatestVersion?: number;
|
||||
/** Internal authenticated refresh write-back only; never accepted by HTTP schemas. */
|
||||
preserveAiSessionEpoch?: boolean;
|
||||
},
|
||||
actor?: { userId?: string | null; agentId?: string | null },
|
||||
) {
|
||||
@@ -2534,6 +2536,7 @@ export function secretService(db: Db | DbTransaction) {
|
||||
.update(companySecrets)
|
||||
.set({
|
||||
latestVersion: nextVersion,
|
||||
aiSessionEpoch: input.preserveAiSessionEpoch ? secret.aiSessionEpoch : sql`${companySecrets.aiSessionEpoch} + 1`,
|
||||
externalRef: prepared.externalRef,
|
||||
providerConfigId,
|
||||
lastRotatedAt: new Date(),
|
||||
@@ -4589,6 +4592,8 @@ export function secretService(db: Db | DbTransaction) {
|
||||
providerVersionRef?: string | null;
|
||||
providerConfigId?: string | null;
|
||||
expectedLatestVersion?: number;
|
||||
/** Internal authenticated refresh write-back only; never accepted by HTTP schemas. */
|
||||
preserveAiSessionEpoch?: boolean;
|
||||
},
|
||||
actor?: { userId?: string | null; agentId?: string | null },
|
||||
) => {
|
||||
|
||||
Reference in new issue
Block a user