feat: add kubernetes execution target provisioning

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-05-12 10:03:32 -05:00
1 parent 0808b388ee
commit 4a7a45f960
87 files changed
+15005 -131

No files matched your search

@@ -0,0 +1,187 @@
/**
* M1 End-to-End smoke — CLI services against a real kind cluster
*
* Exercises the EXACT code path the CLI uses:
* clusterConnectionsService.create (paperclipai cluster add …)
* → createExecutionTargetRegistry / registerKubernetesExecutionTargetDriver
* → driver.ensureTenant (provisions namespace on kind)
* → clusterNamespaceBindingsService.record
*
* Run with:
* pnpm -w exec vitest run --config server/vitest.integration.config.ts k8s-cli-end-to-end
*/
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { execSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
createDb,
companies,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
type Db,
} from "@paperclipai/db";
import { clusterConnectionsService } from "../services/cluster-connections.js";
import { clusterTenantPoliciesService } from "../services/cluster-tenant-policies.js";
import { clusterNamespaceBindingsService } from "../services/cluster-namespace-bindings.js";
import { createExecutionTargetRegistry } from "../adapters/execution-target-registry.js";
import { registerKubernetesExecutionTargetDriver } from "../adapters/execution-targets/kubernetes.js";
// ---------------------------------------------------------------------------
// Inline kind harness (mirrors packages/adapters/kubernetes-execution/test/integration/_harness.ts)
// ---------------------------------------------------------------------------
interface KindCluster {
name: string;
kubeconfigPath: string;
kubeconfigYaml: string;
cleanup(): void;
}
function spinUpKind(): KindCluster {
const name = `pp-e2e-${Math.random().toString(36).slice(2, 8)}`;
const dir = mkdtempSync(join(tmpdir(), `${name}-`));
const kubeconfigPath = join(dir, "kubeconfig");
execSync(`kind create cluster --name ${name} --kubeconfig ${kubeconfigPath} --wait 90s`, {
stdio: "inherit",
});
const kubeconfigYaml = readFileSync(kubeconfigPath, "utf-8");
return {
name,
kubeconfigPath,
kubeconfigYaml,
cleanup: () => {
try { execSync(`kind delete cluster --name ${name}`, { stdio: "ignore" }); } catch { /* swallow */ }
try { rmSync(dir, { recursive: true, force: true }); } catch { /* swallow */ }
},
};
}
function commandExists(command: string): boolean {
try {
execSync(`command -v ${command}`, { stdio: "ignore" });
return true;
} catch {
return false;
}
}
// ---------------------------------------------------------------------------
// Suite-level setup
// ---------------------------------------------------------------------------
let kc: KindCluster;
let dbHandle: EmbeddedPostgresTestDatabase;
let db: Db;
let companyId: string;
const describeWithKind = commandExists("kind") ? describe : describe.skip;
// ---------------------------------------------------------------------------
// Test
// ---------------------------------------------------------------------------
describeWithKind("M1 end-to-end smoke (CLI services + kind)", () => {
beforeAll(async () => {
// Spin up the kind cluster and embedded postgres in parallel.
const [kindResult, dbResult] = await Promise.all([
Promise.resolve(spinUpKind()),
startEmbeddedPostgresTestDatabase("paperclip-m1-e2e-"),
]);
kc = kindResult;
dbHandle = dbResult;
db = createDb(dbHandle.connectionString);
// Seed a company row (required FK for namespace bindings).
const [row] = await db.insert(companies).values({ name: "Acme Corp" }).returning();
companyId = row.id;
}, 240_000);
afterAll(async () => {
kc?.cleanup();
await dbHandle?.cleanup();
});
it(
"registers cluster, provisions a tenant, records binding, asserts idempotency",
async () => {
// 1. Wire services exactly the way the CLI does.
const cs = clusterConnectionsService(db, {
resolveSecret: async () => kc.kubeconfigYaml,
});
const tps = clusterTenantPoliciesService(db);
const bindings = clusterNamespaceBindingsService(db);
// 2. Register a cluster connection (paperclipai cluster add …).
const conn = await cs.create({
label: "smoke-kind",
kind: "kubeconfig",
kubeconfigSecretRef: { provider: "stub", name: "kc" },
capabilities: { cilium: false, storageClass: "standard", architectures: ["amd64"] },
createdBy: "smoke",
});
expect(conn.id).toBeDefined();
expect(conn.label).toBe("smoke-kind");
// 3. Wire the registry exactly like the server does at startup.
const reg = createExecutionTargetRegistry();
registerKubernetesExecutionTargetDriver(reg, {
resolveConnection: (id) => cs.resolve(id),
});
const driver = reg.get("kubernetes")!;
expect(driver).not.toBeNull();
// 4. Ensure the tenant namespace (provisions namespace on kind).
const result = await driver.ensureTenant({
clusterConnectionId: conn.id,
company: { id: companyId, slug: "acme-corp" },
tenantPolicy: null,
driverServiceAccount: { name: "default", namespace: "default" },
controlPlane: { topology: "cross-cluster", namespaceLabels: {}, podLabels: {} },
adapterAllowFqdns: [],
imagePullDockerConfigJson: null,
});
expect(result.namespace).toBe("paperclip-acme-corp");
expect(result.ciliumApplied).toBe(false);
// 5. Record the namespace binding (what the CLI does after ensure-tenant).
await bindings.record({
clusterConnectionId: conn.id,
companyId,
namespaceName: result.namespace,
});
// 6. Verify by reading the binding back from DB.
const binding = await bindings.getByClusterAndCompany(conn.id, companyId);
expect(binding).not.toBeNull();
expect(binding!.namespaceName).toBe("paperclip-acme-corp");
// 7. Verify the tenant policy get path returns null (no policy seeded).
const tp = await tps.get(conn.id, companyId);
expect(tp).toBeNull();
// 8. Idempotency: re-running ensure-tenant must return the same namespace
// without error (namespace already exists on kind).
const result2 = await driver.ensureTenant({
clusterConnectionId: conn.id,
company: { id: companyId, slug: "acme-corp" },
tenantPolicy: null,
driverServiceAccount: { name: "default", namespace: "default" },
controlPlane: { topology: "cross-cluster", namespaceLabels: {}, podLabels: {} },
adapterAllowFqdns: [],
imagePullDockerConfigJson: null,
});
expect(result2.namespace).toBe(result.namespace);
// 9. Record the binding again; idempotent upsert should not throw.
await bindings.record({
clusterConnectionId: conn.id,
companyId,
namespaceName: result2.namespace,
});
const binding2 = await bindings.getByClusterAndCompany(conn.id, companyId);
expect(binding2!.namespaceName).toBe("paperclip-acme-corp");
},
240_000,
);
});
@@ -0,0 +1,39 @@
import { describe, it, expect } from "vitest";
import { createExecutionTargetRegistry } from "./execution-target-registry.js";
import type { ExecutionTargetDriver } from "./execution-target-registry.js";
function fakeDriver(): ExecutionTargetDriver {
return {
type: "kubernetes",
validateTarget: async () => {},
ensureTenant: async () => ({ namespace: "x", ciliumApplied: false }),
run: async () => ({ exitCode: 0, signal: null, timedOut: false }),
};
}
describe("ExecutionTargetDriverRegistry", () => {
it("registers and retrieves a driver by kind", () => {
const reg = createExecutionTargetRegistry();
const d = fakeDriver();
reg.register(d);
expect(reg.get("kubernetes")).toBe(d);
});
it("rejects duplicate registrations of the same kind", () => {
const reg = createExecutionTargetRegistry();
reg.register(fakeDriver());
expect(() => reg.register(fakeDriver())).toThrow(/already registered/i);
});
it("returns null for unknown kinds", () => {
const reg = createExecutionTargetRegistry();
expect(reg.get("kubernetes")).toBeNull();
});
it("list() returns all registered drivers", () => {
const reg = createExecutionTargetRegistry();
const d = fakeDriver();
reg.register(d);
expect(reg.list()).toEqual([d]);
});
});
@@ -0,0 +1,31 @@
import type { KubernetesExecutionDriver } from "@paperclipai/execution-target-kubernetes";
/**
* Discriminated union of every registered execution-target driver.
* Future kinds (Nomad, ECS, Modal, ...) extend this union.
*/
export type ExecutionTargetDriver = KubernetesExecutionDriver;
export interface ExecutionTargetDriverRegistry {
register(driver: ExecutionTargetDriver): void;
get(kind: ExecutionTargetDriver["type"]): ExecutionTargetDriver | null;
list(): ExecutionTargetDriver[];
}
export function createExecutionTargetRegistry(): ExecutionTargetDriverRegistry {
const drivers = new Map<string, ExecutionTargetDriver>();
return {
register(driver) {
if (drivers.has(driver.type)) {
throw new Error(`Execution target driver "${driver.type}" already registered`);
}
drivers.set(driver.type, driver);
},
get(kind) {
return drivers.get(kind) ?? null;
},
list() {
return [...drivers.values()];
},
};
}
@@ -0,0 +1,25 @@
import { describe, it, expect } from "vitest";
import { registerKubernetesExecutionTargetDriver } from "./kubernetes.js";
import { createExecutionTargetRegistry } from "../execution-target-registry.js";
describe("registerKubernetesExecutionTargetDriver", () => {
it("registers a driver of type 'kubernetes' into the registry", () => {
const reg = createExecutionTargetRegistry();
registerKubernetesExecutionTargetDriver(reg, {
resolveConnection: async () => null,
});
const drv = reg.get("kubernetes");
expect(drv).not.toBeNull();
expect(drv?.type).toBe("kubernetes");
});
it("a freshly registered driver rejects unknown connection ids", async () => {
const reg = createExecutionTargetRegistry();
registerKubernetesExecutionTargetDriver(reg, {
resolveConnection: async () => null,
});
const drv = reg.get("kubernetes")!;
await expect(drv.validateTarget({ kind: "kubernetes", clusterConnectionId: "missing" }))
.rejects.toThrow(/not found/i);
});
});
@@ -0,0 +1,17 @@
import {
createKubernetesExecutionDriver,
type KubernetesDriverDeps,
} from "@paperclipai/execution-target-kubernetes";
import type { ExecutionTargetDriverRegistry } from "../execution-target-registry.js";
/**
* Wires the Kubernetes execution-target driver into the registry. Called once
* at server startup with a closure that resolves cluster connection records
* (kubeconfig blob via Paperclip secret store, etc.) to ResolvedClusterConnection.
*/
export function registerKubernetesExecutionTargetDriver(
registry: ExecutionTargetDriverRegistry,
deps: KubernetesDriverDeps,
): void {
registry.register(createKubernetesExecutionDriver(deps));
}
+35
View File
@@ -48,6 +48,10 @@ import type {
InstanceDatabaseBackupRunResult,
InstanceDatabaseBackupTrigger,
} from "./routes/instance-database-backups.js";
import { createExecutionTargetRegistry } from "./adapters/execution-target-registry.js";
import { registerKubernetesExecutionTargetDriver } from "./adapters/execution-targets/kubernetes.js";
import { clusterConnectionsService } from "./services/cluster-connections.js";
import { getSecretProvider } from "./secrets/provider-registry.js";
type BetterAuthSessionUser = {
id: string;
@@ -593,6 +597,37 @@ export async function startServer(): Promise<StartedServer> {
}
};
const pluginWorkerManager = createPluginWorkerManager();
// ---------------------------------------------------------------------------
// Execution-target registry
//
// The kubeconfigSecretRef stored in cluster_connections uses
// { provider, name } where `name` is the externalRef (for external providers
// such as AWS Secrets Manager / GCP Secret Manager / Vault). For those
// providers, `resolveVersion({ material: {}, externalRef: name })` is the
// correct call. For the `local_encrypted` provider the kubeconfig blob must
// be stored via an external provider — instance-level local-encrypted storage
// is not yet wired (M2 gap: design a cluster-scoped secret vault or let
// operators use an external provider for kubeconfig credentials).
// ---------------------------------------------------------------------------
const clusterConnections = clusterConnectionsService(db as any, {
resolveSecret: async (ref) => {
const provider = getSecretProvider(ref.provider as Parameters<typeof getSecretProvider>[0]);
// `ref.name` is the externalRef used by external secret providers
// (AWS SM / GCP SM / Vault). For `local_encrypted`, kubeconfig material
// must be stored via an external provider; calling this path with
// `local_encrypted` will throw from the provider itself.
// TODO(M2): design instance-scoped secret storage for local_encrypted
// kubeconfig blobs so operators without external providers
// can still register clusters.
return provider.resolveVersion({ material: {}, externalRef: ref.name });
},
});
const executionTargetRegistry = createExecutionTargetRegistry();
registerKubernetesExecutionTargetDriver(executionTargetRegistry, {
resolveConnection: (id) => clusterConnections.resolve(id),
});
const app = await createApp(db as any, {
uiMode,
serverPort: listenPort,
@@ -0,0 +1,76 @@
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { startEmbeddedPostgresTestDatabase, type EmbeddedPostgresTestDatabase, createDb } from "@paperclipai/db";
import type { Db } from "@paperclipai/db";
import { clusterConnectionsService } from "./cluster-connections.js";
let dbHandle: EmbeddedPostgresTestDatabase;
let db: Db;
beforeAll(async () => {
dbHandle = await startEmbeddedPostgresTestDatabase("paperclip-cluster-conn-test-");
db = createDb(dbHandle.connectionString);
}, 60_000);
afterAll(async () => { await dbHandle?.cleanup(); });
describe("clusterConnectionsService", () => {
it("creates, lists, gets, resolves, and deletes a connection", async () => {
const svc = clusterConnectionsService(db, {
resolveSecret: async (ref) => `fake-kubeconfig-yaml:${ref.provider}:${ref.name}`,
});
const created = await svc.create({
label: "kind-test",
kind: "kubeconfig",
kubeconfigSecretRef: { provider: "local_encrypted", name: "kind-cfg" },
capabilities: { cilium: false, storageClass: "standard", architectures: ["amd64"] },
createdBy: "system",
});
expect(created.id).toBeDefined();
expect(created.defaultNamespacePrefix).toBe("paperclip-");
expect(created.allowAgentImageOverride).toBe(false);
const list = await svc.list();
expect(list).toHaveLength(1);
const fetched = await svc.get(created.id);
expect(fetched?.label).toBe("kind-test");
const resolved = await svc.resolve(created.id);
expect(resolved?.kubeconfigYaml).toBe("fake-kubeconfig-yaml:local_encrypted:kind-cfg");
await svc.delete(created.id);
expect(await svc.list()).toHaveLength(0);
});
it("rejects duplicate labels", async () => {
const svc = clusterConnectionsService(db, { resolveSecret: async () => "x" });
await svc.create({
label: "dupe", kind: "in-cluster",
capabilities: { cilium: false, storageClass: "standard", architectures: ["amd64"] },
createdBy: "system",
});
await expect(svc.create({
label: "dupe", kind: "in-cluster",
capabilities: { cilium: false, storageClass: "standard", architectures: ["amd64"] },
createdBy: "system",
})).rejects.toThrow(/label/i);
});
it("resolve() returns null for an unknown id", async () => {
const svc = clusterConnectionsService(db, { resolveSecret: async () => "x" });
expect(await svc.resolve("00000000-0000-0000-0000-000000000000")).toBeNull();
});
it("resolve() omits kubeconfigYaml for in-cluster connections", async () => {
const svc = clusterConnectionsService(db, {
resolveSecret: async () => { throw new Error("should not be called for in-cluster"); },
});
const created = await svc.create({
label: "in-cluster-test", kind: "in-cluster",
capabilities: { cilium: false, storageClass: "standard", architectures: ["amd64"] },
createdBy: "system",
});
const resolved = await svc.resolve(created.id);
expect(resolved?.kind).toBe("in-cluster");
expect(resolved?.kubeconfigYaml).toBeUndefined();
});
});
+123
View File
@@ -0,0 +1,123 @@
import { eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { clusterConnections } from "@paperclipai/db";
import type { ResolvedClusterConnection, ClusterCapabilities } from "@paperclipai/execution-target-kubernetes";
export interface ClusterConnectionRow {
id: string;
label: string;
kind: "in-cluster" | "kubeconfig";
kubeconfigSecretRef: { provider: string; name: string } | null;
apiServerUrl: string | null;
defaultNamespacePrefix: string;
capabilities: ClusterCapabilities;
paperclipPublicUrl: string | null;
imageRegistry: string | null;
allowAgentImageOverride: boolean;
createdAt: Date;
createdBy: string;
}
export interface CreateClusterConnectionInput {
label: string;
kind: "in-cluster" | "kubeconfig";
kubeconfigSecretRef?: { provider: string; name: string };
apiServerUrl?: string;
defaultNamespacePrefix?: string;
capabilities: ClusterCapabilities;
paperclipPublicUrl?: string;
imageRegistry?: string;
allowAgentImageOverride?: boolean;
createdBy: string;
}
export interface ClusterConnectionsServiceDeps {
resolveSecret: (ref: { provider: string; name: string }) => Promise<string>;
}
export interface ClusterConnectionsService {
create(input: CreateClusterConnectionInput): Promise<ClusterConnectionRow>;
list(): Promise<ClusterConnectionRow[]>;
get(id: string): Promise<ClusterConnectionRow | null>;
delete(id: string): Promise<void>;
resolve(id: string): Promise<ResolvedClusterConnection | null>;
}
export function clusterConnectionsService(db: Db, deps: ClusterConnectionsServiceDeps): ClusterConnectionsService {
return {
async create(input) {
try {
const [row] = await db.insert(clusterConnections).values({
label: input.label,
kind: input.kind,
kubeconfigSecretRef: input.kubeconfigSecretRef ?? null,
apiServerUrl: input.apiServerUrl ?? null,
defaultNamespacePrefix: input.defaultNamespacePrefix ?? "paperclip-",
capabilities: input.capabilities,
paperclipPublicUrl: input.paperclipPublicUrl ?? null,
imageRegistry: input.imageRegistry ?? null,
allowAgentImageOverride: input.allowAgentImageOverride ?? false,
createdBy: input.createdBy,
}).returning();
return mapRow(row);
} catch (err) {
if (/cluster_connections_label_uq/.test(String(err))) {
throw new Error(`A cluster connection with label "${input.label}" already exists`);
}
throw err;
}
},
async list() {
const rows = await db.select().from(clusterConnections);
return rows.map(mapRow);
},
async get(id) {
const [row] = await db.select().from(clusterConnections).where(eq(clusterConnections.id, id));
return row ? mapRow(row) : null;
},
async delete(id) {
await db.delete(clusterConnections).where(eq(clusterConnections.id, id));
},
async resolve(id) {
const row = await this.get(id);
if (!row) return null;
let kubeconfigYaml: string | undefined;
if (row.kind === "kubeconfig" && row.kubeconfigSecretRef) {
kubeconfigYaml = await deps.resolveSecret(row.kubeconfigSecretRef);
}
return {
id: row.id,
label: row.label,
kind: row.kind,
kubeconfigYaml,
apiServerUrl: row.apiServerUrl,
defaultNamespacePrefix: row.defaultNamespacePrefix,
paperclipPublicUrl: row.paperclipPublicUrl,
imageRegistry: row.imageRegistry,
allowAgentImageOverride: row.allowAgentImageOverride,
capabilities: row.capabilities,
};
},
};
}
function mapRow(row: typeof clusterConnections.$inferSelect): ClusterConnectionRow {
return {
id: row.id,
label: row.label,
kind: row.kind as "in-cluster" | "kubeconfig",
kubeconfigSecretRef: row.kubeconfigSecretRef ?? null,
apiServerUrl: row.apiServerUrl ?? null,
defaultNamespacePrefix: row.defaultNamespacePrefix,
capabilities: row.capabilities as ClusterCapabilities,
paperclipPublicUrl: row.paperclipPublicUrl ?? null,
imageRegistry: row.imageRegistry ?? null,
allowAgentImageOverride: row.allowAgentImageOverride,
createdAt: row.createdAt,
createdBy: row.createdBy,
};
}
@@ -0,0 +1,130 @@
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import {
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
createDb,
} from "@paperclipai/db";
import type { Db } from "@paperclipai/db";
import { sql } from "drizzle-orm";
import { clusterNamespaceBindingsService } from "./cluster-namespace-bindings.js";
let dbHandle: EmbeddedPostgresTestDatabase;
let db: Db;
let clusterId: string;
let companyId: string;
beforeAll(async () => {
dbHandle = await startEmbeddedPostgresTestDatabase("paperclip-ns-binding-test-");
db = createDb(dbHandle.connectionString);
// Seed a cluster connection
const clusterRows = await db.execute(sql`
INSERT INTO cluster_connections (label, kind, capabilities, created_by)
VALUES ('seed-cluster', 'in-cluster', '{"cilium":false,"storageClass":"standard","architectures":["amd64"]}'::jsonb, 'sys')
RETURNING id
`);
clusterId = (clusterRows[0] as { id: string }).id;
// Seed a company
const companyRows = await db.execute(sql`
INSERT INTO companies (name)
VALUES ('Acme')
RETURNING id
`);
companyId = (companyRows[0] as { id: string }).id;
}, 60_000);
afterAll(async () => {
await dbHandle?.cleanup();
});
describe("clusterNamespaceBindingsService", () => {
it("getByClusterAndCompany() returns null when no binding exists", async () => {
const svc = clusterNamespaceBindingsService(db);
const result = await svc.getByClusterAndCompany(clusterId, companyId);
expect(result).toBeNull();
});
it("record() creates a new binding", async () => {
const svc = clusterNamespaceBindingsService(db);
await svc.record({
clusterConnectionId: clusterId,
companyId,
namespaceName: "paperclip-acme",
});
const found = await svc.getByClusterAndCompany(clusterId, companyId);
expect(found).not.toBeNull();
expect(found?.namespaceName).toBe("paperclip-acme");
});
it("record() is idempotent — second call updates the namespace name", async () => {
const svc = clusterNamespaceBindingsService(db);
await svc.record({
clusterConnectionId: clusterId,
companyId,
namespaceName: "paperclip-acme-v2",
});
const found = await svc.getByClusterAndCompany(clusterId, companyId);
expect(found?.namespaceName).toBe("paperclip-acme-v2");
});
it("record() handles concurrent first-write without violating the unique constraint", async () => {
// Seed an isolated company so this test does not interact with the rows
// written by the earlier suite cases.
const conRows = await db.execute(sql`
INSERT INTO companies (name, issue_prefix)
VALUES ('Gamma Corp', 'GAM')
RETURNING id
`);
const conCompanyId = (conRows[0] as { id: string }).id;
const svc = clusterNamespaceBindingsService(db);
const N = 8;
// The pre-fix select-then-insert shape lost this race: two callers would
// both observe no row and one would hit the unique constraint as an
// unhandled error. With ON CONFLICT the upsert is unconditionally atomic.
await Promise.all(
Array.from({ length: N }, (_, i) =>
svc.record({
clusterConnectionId: clusterId,
companyId: conCompanyId,
namespaceName: `paperclip-gamma-${i}`,
}),
),
);
const found = await svc.getByClusterAndCompany(clusterId, conCompanyId);
expect(found).not.toBeNull();
// The winning name is whichever upsert committed last; we only need to
// assert that exactly one row exists and no exceptions were thrown.
expect(found?.namespaceName).toMatch(/^paperclip-gamma-\d$/);
});
it("record() preserves other rows when called for a different company", async () => {
const svc = clusterNamespaceBindingsService(db);
// Seed a second company
const otherRows = await db.execute(sql`
INSERT INTO companies (name, issue_prefix)
VALUES ('Beta Corp', 'BET')
RETURNING id
`);
const otherCompanyId = (otherRows[0] as { id: string }).id;
await svc.record({
clusterConnectionId: clusterId,
companyId: otherCompanyId,
namespaceName: "paperclip-beta-corp",
});
// Original binding should still be paperclip-acme-v2
const acme = await svc.getByClusterAndCompany(clusterId, companyId);
expect(acme?.namespaceName).toBe("paperclip-acme-v2");
// New binding should be paperclip-beta-corp
const beta = await svc.getByClusterAndCompany(clusterId, otherCompanyId);
expect(beta?.namespaceName).toBe("paperclip-beta-corp");
});
});
@@ -0,0 +1,56 @@
import { and, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { clusterNamespaceBindings } from "@paperclipai/db";
export interface RecordBindingInput {
clusterConnectionId: string;
companyId: string;
namespaceName: string;
}
export interface ClusterNamespaceBindingsService {
record(input: RecordBindingInput): Promise<void>;
getByClusterAndCompany(
clusterConnectionId: string,
companyId: string,
): Promise<{ id: string; namespaceName: string } | null>;
}
export function clusterNamespaceBindingsService(db: Db): ClusterNamespaceBindingsService {
return {
async record(input) {
// Atomic upsert keyed on the (cluster_connection_id, company_id) unique
// index. The previous select-then-insert had a TOCTOU race under
// concurrent provisioning: two callers could both observe no existing
// binding, both INSERT, and one would hit the unique constraint as an
// unhandled error.
await db
.insert(clusterNamespaceBindings)
.values({
clusterConnectionId: input.clusterConnectionId,
companyId: input.companyId,
namespaceName: input.namespaceName,
})
.onConflictDoUpdate({
target: [clusterNamespaceBindings.clusterConnectionId, clusterNamespaceBindings.companyId],
set: {
namespaceName: input.namespaceName,
updatedAt: sql`now()`,
},
});
},
async getByClusterAndCompany(clusterConnectionId, companyId) {
const [row] = await db
.select()
.from(clusterNamespaceBindings)
.where(
and(
eq(clusterNamespaceBindings.clusterConnectionId, clusterConnectionId),
eq(clusterNamespaceBindings.companyId, companyId),
),
);
return row ? { id: row.id, namespaceName: row.namespaceName } : null;
},
};
}
@@ -0,0 +1,134 @@
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { startEmbeddedPostgresTestDatabase, type EmbeddedPostgresTestDatabase, createDb } from "@paperclipai/db";
import type { Db } from "@paperclipai/db";
import { sql } from "drizzle-orm";
import { clusterTenantPoliciesService } from "./cluster-tenant-policies.js";
let dbHandle: EmbeddedPostgresTestDatabase;
let db: Db;
let clusterId: string;
let companyId: string;
beforeAll(async () => {
dbHandle = await startEmbeddedPostgresTestDatabase("paperclip-tenant-pol-test-");
db = createDb(dbHandle.connectionString);
// Seed a cluster connection and a company.
const clusterRows = await db.execute(sql`
INSERT INTO cluster_connections (label, kind, capabilities, created_by)
VALUES ('seed-cluster', 'in-cluster', '{"cilium":false,"storageClass":"standard","architectures":["amd64"]}'::jsonb, 'sys')
RETURNING id
`);
clusterId = (clusterRows[0] as { id: string }).id;
// companies has: id (uuid, defaultRandom), name (text notNull), and everything else defaults.
const companyRows = await db.execute(sql`
INSERT INTO companies (name)
VALUES ('Acme')
RETURNING id
`);
companyId = (companyRows[0] as { id: string }).id;
}, 60_000);
afterAll(async () => { await dbHandle?.cleanup(); });
describe("clusterTenantPoliciesService", () => {
it("get() returns null when no policy exists", async () => {
const svc = clusterTenantPoliciesService(db);
const result = await svc.get(clusterId, companyId);
expect(result).toBeNull();
});
it("upsert() creates a row, then a second upsert updates it", async () => {
const svc = clusterTenantPoliciesService(db);
const created = await svc.upsert({
clusterConnectionId: clusterId,
companyId,
quota: { requestsCpu: "32" },
limitRange: null,
additionalAllowFqdns: ["api.acme.io"],
imageOverrides: null,
});
expect(created.quota?.requestsCpu).toBe("32");
expect(created.additionalAllowFqdns).toEqual(["api.acme.io"]);
const updated = await svc.upsert({
clusterConnectionId: clusterId,
companyId,
quota: { requestsCpu: "64", countJobs: 200 },
limitRange: { default: { cpu: "2" } },
additionalAllowFqdns: ["api.acme.io", "*.linear.app"],
imageOverrides: { claude_local: "ecr.acme.io/agent-runtime-claude:v1" },
});
expect(updated.quota?.requestsCpu).toBe("64");
expect(updated.quota?.countJobs).toBe(200);
expect(updated.additionalAllowFqdns).toEqual(["api.acme.io", "*.linear.app"]);
expect(updated.imageOverrides?.claude_local).toBe("ecr.acme.io/agent-runtime-claude:v1");
});
it("get() reads back what upsert wrote", async () => {
const svc = clusterTenantPoliciesService(db);
const fetched = await svc.get(clusterId, companyId);
expect(fetched).not.toBeNull();
expect(fetched?.additionalAllowFqdns).toEqual(["api.acme.io", "*.linear.app"]);
expect(fetched?.limitRange?.default?.cpu).toBe("2");
});
it("upsert() handles concurrent first-write without a unique-constraint error", async () => {
// Seed an isolated company so this case doesn't interact with rows
// written by earlier suite cases.
const conRows = await db.execute(sql`
INSERT INTO companies (name, issue_prefix) VALUES ('Delta Corp', 'DLT') RETURNING id
`);
const conCompanyId = (conRows[0] as { id: string }).id;
const svc = clusterTenantPoliciesService(db);
const N = 8;
// Pre-fix shape lost this race: two callers both observed existing=null
// and both attempted INSERT, with the loser surfacing a unique-constraint
// error from cluster_tenant_policies_cluster_company_uq. The atomic
// ON CONFLICT shape eliminates that error path.
await Promise.all(
Array.from({ length: N }, (_, i) =>
svc.upsert({
clusterConnectionId: clusterId,
companyId: conCompanyId,
quota: { requestsCpu: `${i}` },
limitRange: null,
additionalAllowFqdns: [],
imageOverrides: null,
}),
),
);
const found = await svc.get(clusterId, conCompanyId);
expect(found).not.toBeNull();
expect(found?.quota?.requestsCpu).toMatch(/^\d$/);
});
it("upsert() preserves httpProxyUrl when the caller doesn't pass one", async () => {
const svc = clusterTenantPoliciesService(db);
// Set a proxy URL via an explicit upsert.
await svc.upsert({
clusterConnectionId: clusterId, companyId,
quota: null, limitRange: null,
additionalAllowFqdns: [], imageOverrides: null,
httpProxyUrl: "http://proxy.acme.internal:3128",
});
expect((await svc.get(clusterId, companyId))?.httpProxyUrl).toBe("http://proxy.acme.internal:3128");
// Subsequent upsert without httpProxyUrl must NOT clear it.
await svc.upsert({
clusterConnectionId: clusterId, companyId,
quota: null, limitRange: null,
additionalAllowFqdns: ["api.example.com"], imageOverrides: null,
});
expect((await svc.get(clusterId, companyId))?.httpProxyUrl).toBe("http://proxy.acme.internal:3128");
// Explicit null clears it.
await svc.upsert({
clusterConnectionId: clusterId, companyId,
quota: null, limitRange: null,
additionalAllowFqdns: [], imageOverrides: null,
httpProxyUrl: null,
});
expect((await svc.get(clusterId, companyId))?.httpProxyUrl).toBeNull();
});
});
@@ -0,0 +1,91 @@
import { and, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { clusterTenantPolicies } from "@paperclipai/db";
import type { TenantPolicy } from "@paperclipai/execution-target-kubernetes";
export interface UpsertTenantPolicyInput {
clusterConnectionId: string;
companyId: string;
quota: TenantPolicy["quota"];
limitRange: TenantPolicy["limitRange"];
additionalAllowFqdns: string[];
/**
* Egress HTTP proxy URL for the tenant. When omitted (undefined), any existing
* value is preserved on upsert; pass `null` explicitly to clear it.
*/
httpProxyUrl?: string | null;
imageOverrides: Record<string, string> | null;
}
export interface TenantPolicyRow extends TenantPolicy {
clusterConnectionId: string;
companyId: string;
httpProxyUrl: string | null;
}
export interface ClusterTenantPoliciesService {
get(clusterConnectionId: string, companyId: string): Promise<TenantPolicyRow | null>;
upsert(input: UpsertTenantPolicyInput): Promise<TenantPolicyRow>;
}
export function clusterTenantPoliciesService(db: Db): ClusterTenantPoliciesService {
return {
async get(clusterConnectionId, companyId) {
const [row] = await db.select().from(clusterTenantPolicies).where(and(
eq(clusterTenantPolicies.clusterConnectionId, clusterConnectionId),
eq(clusterTenantPolicies.companyId, companyId),
));
return row ? mapRow(row) : null;
},
async upsert(input) {
// We still read existing once to implement the preserve-on-omit
// semantics for httpProxyUrl. The read is racy w.r.t. another concurrent
// upsert, but the write below is unconditionally atomic via
// ON CONFLICT — no caller will surface a unique-constraint error from
// cluster_tenant_policies_cluster_company_uq. Race semantics under
// concurrent (omit, set, set, omit, …) call patterns: an explicit
// setter always overwrites an omitter's preserved value, which is the
// desired last-explicit-write-wins shape.
const existing = await this.get(input.clusterConnectionId, input.companyId);
const httpProxyUrl =
input.httpProxyUrl === undefined ? (existing?.httpProxyUrl ?? null) : input.httpProxyUrl;
const networkJson = { additionalAllowFqdns: input.additionalAllowFqdns, httpProxyUrl };
const [row] = await db
.insert(clusterTenantPolicies)
.values({
clusterConnectionId: input.clusterConnectionId,
companyId: input.companyId,
quotaJson: input.quota,
limitRangeJson: input.limitRange,
networkJson,
imageOverridesJson: input.imageOverrides,
})
.onConflictDoUpdate({
target: [clusterTenantPolicies.clusterConnectionId, clusterTenantPolicies.companyId],
set: {
quotaJson: input.quota,
limitRangeJson: input.limitRange,
networkJson,
imageOverridesJson: input.imageOverrides,
updatedAt: sql`now()`,
},
})
.returning();
return mapRow(row);
},
};
}
function mapRow(r: typeof clusterTenantPolicies.$inferSelect): TenantPolicyRow {
return {
clusterConnectionId: r.clusterConnectionId,
companyId: r.companyId,
quota: r.quotaJson ?? null,
limitRange: r.limitRangeJson ?? null,
additionalAllowFqdns: r.networkJson?.additionalAllowFqdns ?? [],
httpProxyUrl: r.networkJson?.httpProxyUrl ?? null,
imageOverrides: r.imageOverridesJson ?? null,
};
}