mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
merge: integrate agent lifecycle ownership from master
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
commit
ad5d8826ad
159 files changed
+7173
-2805
No files matched your search
@@ -0,0 +1,68 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createLifecycleWorker } from "../application/worker.js";
|
||||
import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../application/ports.js";
|
||||
import { startAgentLifecycle } from "../index.js";
|
||||
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
|
||||
import { logger } from "../../../middleware/logger.js";
|
||||
|
||||
const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3,
|
||||
lifecycleRequiredPluginIds: null, lifecycleError: null,
|
||||
lifecycleOperation: { id: "operation", hostComplete: false, completedPluginIds: [], attempts: 1 } };
|
||||
function fixture() {
|
||||
const store = { get: vi.fn(), claim: vi.fn().mockResolvedValue(agent), renew: vi.fn().mockResolvedValue(true),
|
||||
setRequiredPlugins: vi.fn().mockResolvedValue(true), completeHost: vi.fn().mockResolvedValue(true),
|
||||
completePlugin: vi.fn().mockResolvedValue(true), defer: vi.fn().mockResolvedValue(true),
|
||||
change: vi.fn().mockResolvedValue({ ...agent, lifecycleState: "ready" }), pending: vi.fn().mockResolvedValue([agent.id]) } satisfies LifecycleStore;
|
||||
const driver = { requiredPluginIds: vi.fn().mockResolvedValue(["plugin"]), runHost: vi.fn().mockResolvedValue("complete"),
|
||||
runPlugin: vi.fn().mockResolvedValue("complete") } satisfies LifecycleDriver;
|
||||
const report = vi.fn();
|
||||
return { store, driver, report, worker: createLifecycleWorker(store, driver, report) };
|
||||
}
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
|
||||
|
||||
describe("lifecycle failure diagnostics", () => {
|
||||
it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => {
|
||||
const { store, driver, report, worker } = fixture();
|
||||
const error = new Error("private provider detail");
|
||||
if (stage === "host") driver.runHost.mockRejectedValue(error);
|
||||
else if (stage === "plugin") driver.runPlugin.mockRejectedValue(error);
|
||||
else if (stage === "scan") store.pending.mockRejectedValue(error);
|
||||
else if (stage === "defer") { driver.runHost.mockResolvedValue("pending"); store.defer.mockRejectedValue(error); }
|
||||
else store[stage].mockRejectedValue(error);
|
||||
const work = stage === "scan" ? worker.sweep() : worker.process(agent.id);
|
||||
if (["claim", "scan", "defer"].includes(stage)) await expect(work).rejects.toThrow(error);
|
||||
else await work;
|
||||
expect(report).toHaveBeenCalledWith(expect.objectContaining({ stage }), error);
|
||||
if (!["claim", "scan"].includes(stage)) expect(report).toHaveBeenCalledWith(expect.objectContaining({
|
||||
agentId: agent.id, companyId: agent.companyId, phase: "verifying", operationId: "operation", version: 3,
|
||||
...(stage === "plugin" ? { pluginId: "plugin" } : {}),
|
||||
}), error);
|
||||
await worker.stop();
|
||||
});
|
||||
|
||||
it("reports periodic policy scan failures", async () => {
|
||||
vi.useFakeTimers();
|
||||
const error = new Error("private database detail");
|
||||
const db = { select: () => { throw error; } } as unknown as Db;
|
||||
const effects = { ...createAgentLifecycleEffects(), reportFailure: vi.fn() };
|
||||
const work = startAgentLifecycle(db, effects, fixture().driver, () => true);
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(effects.reportFailure).toHaveBeenCalledWith({ stage: "policy_scan" }, error);
|
||||
} finally { await work.stop(); }
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"],
|
||||
[{ code: "harness_test_failed" }, "harness_test_failed"],
|
||||
[{ cause: { code: "40P01", message: "private query" } }, "40P01"],
|
||||
[{ code: "private credential value" }, "unclassified"],
|
||||
])("logs known codes without provider error contents", (details, failureCode) => {
|
||||
const log = vi.spyOn(logger, "warn").mockImplementation(() => {});
|
||||
const context = { stage: "host" as const, agentId: "agent", phase: "verifying" as const, operationId: "operation", version: 3 };
|
||||
createAgentLifecycleEffects().reportFailure(context, Object.assign(new Error("private provider detail"), details));
|
||||
expect(log).toHaveBeenCalledExactlyOnceWith({ ...context, failureCode }, "Agent lifecycle operation failed");
|
||||
expect(JSON.stringify(log.mock.calls)).not.toContain("private");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,673 @@
|
||||
import { createAgentLifecycle as createLifecycleCommands } from "../index.js";
|
||||
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
|
||||
import { deleteCompany } from "../../../services/company-deletion.js";
|
||||
import { agentLifecycleCompanyDeletion } from "../company-deletion.js";
|
||||
import { requireServerAdapter } from "../../../adapters/index.js";
|
||||
import { agentExecutionsHaveStopped } from "../../../services/agent-execution-stop.js";
|
||||
import { remoteTerminationReceipt } from "../../../services/remote-execution-termination.js";
|
||||
import { approvalService } from "../../../services/approvals.js";
|
||||
import { updateAgentConfigurationInTransaction } from "../../../services/agent-configuration-transaction.js";
|
||||
import { agentService as agentConfiguration } from "../../../services/agents.js";
|
||||
import { readFileSync } from "node:fs";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { agentRoutes } from "../../../routes/agents.js";
|
||||
import { errorHandler } from "../../../middleware/error-handler.js";
|
||||
import { createLifecycleDriver } from "../../../services/agent-lifecycle-driver.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, approvals, projects, issues, heartbeatRuns, nativeRunFinalizations, budgetPolicies, environmentLeases, agentApiKeys, activityLog, agentConfigRevisions, userCompanyPreferences, companies, companyMemberships, principalPermissionGrants, plugins, pluginCompanySettings, createDb, type Db } from "@paperclipai/db";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createAgentLifecycle, configureAgentLifecycle } from "../../../services/agent-lifecycle.js";
|
||||
import { createLifecycleStore as createStore } from "../adapters/postgres.js";
|
||||
import { transition } from "../domain/policy.js";
|
||||
import type { LifecycleAgent } from "../application/ports.js";
|
||||
import { budgetService } from "../../../services/budgets.js";
|
||||
|
||||
const createLifecycleStore = (db: Db) => createStore(db, createAgentLifecycleEffects());
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("agent lifecycle commands", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let companyId: string;
|
||||
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }, 90_000);
|
||||
afterAll(async () => { for (const worker of workers) await worker.stop(); await database?.cleanup(); });
|
||||
beforeEach(async () => {
|
||||
for (const work of workers) await work.stop();
|
||||
companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Lifecycle test", issuePrefix: `L${companyId.slice(0, 7)}` });
|
||||
});
|
||||
function worker(runHost: (agent: LifecycleAgent) => Promise<"complete" | "pending"> = async () => "complete") {
|
||||
const value = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost });
|
||||
workers.push(value); return value;
|
||||
}
|
||||
async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); }
|
||||
async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; }
|
||||
|
||||
it("maps existing states without starting setup or clearing pause reasons", async () => {
|
||||
const rows = await db.insert(agents).values(["idle", "running", "error", "paused", "pending_approval", "terminated"].map(status => ({
|
||||
companyId, name: status, status, pauseReason: status === "paused" ? "budget" : null,
|
||||
}))).returning();
|
||||
const migration = readFileSync(new URL("../../../../../packages/db/src/migrations/0320_daily_onslaught.sql", import.meta.url), "utf8");
|
||||
await db.execute(sql.raw(migration.slice(migration.indexOf('UPDATE "agents"'))));
|
||||
for (const row of rows) {
|
||||
const mapped = await current(row.id);
|
||||
expect(mapped.lifecycleState).toBe(["paused", "pending_approval", "terminated"].includes(row.status) ? row.status : "ready");
|
||||
expect(mapped.lifecycleOperation).toBeNull();
|
||||
expect(mapped.lifecycleRequiredPluginIds).toBeNull();
|
||||
expect(mapped.lifecycleHolds).toEqual(row.status === "paused" ? ["budget"] : []);
|
||||
}
|
||||
});
|
||||
|
||||
it("commits the record before preparation and verifies before admission", async () => {
|
||||
const observed: string[] = [];
|
||||
const work = worker(async agent => {
|
||||
expect((await current(agent.id)).lifecycleState).toBe(agent.lifecycleState);
|
||||
observed.push(agent.lifecycleState); return "complete";
|
||||
});
|
||||
const agent = await hire();
|
||||
expect(agent).toMatchObject({ lifecycleState: "preparing", status: "paused" });
|
||||
await work.process(agent.id);
|
||||
expect(observed).toEqual(["preparing", "verifying"]);
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle" });
|
||||
expect(agent).not.toHaveProperty("lifecycleOperation");
|
||||
});
|
||||
|
||||
it("rolls back a hire and its grants when hire initialization fails", async () => {
|
||||
const effects = createAgentLifecycleEffects();
|
||||
const lifecycle = createLifecycleCommands(db, { ...effects,
|
||||
initializeHire: async (tx, agent, options) => {
|
||||
await effects.initializeHire(tx, agent, options);
|
||||
throw new Error("Hire initialization failed");
|
||||
},
|
||||
});
|
||||
await expect(lifecycle.requestHire(companyId, { name: "Rollback hire", adapterType: "process" }))
|
||||
.rejects.toThrow("Hire initialization failed");
|
||||
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("runs the saved harness test and keeps a failed configuration out of ready", async () => {
|
||||
const driver = createLifecycleDriver(db, {} as never);
|
||||
const work = worker(driver.runHost);
|
||||
const agent = await hire();
|
||||
await work.process(agent.id);
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", status: "paused" });
|
||||
await agentConfiguration(db).update(agent.id, { adapterConfig: { command: process.execPath } });
|
||||
await work.process(agent.id);
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null });
|
||||
});
|
||||
|
||||
it.each(["claude_local", "gemini_local"])("keeps %s authentication warnings out of ready", async adapterType => {
|
||||
const adapter = requireServerAdapter(adapterType);
|
||||
const probe = vi.spyOn(adapter, "testEnvironment").mockResolvedValue({ adapterType, status: "warn",
|
||||
testedAt: new Date().toISOString(), checks: [{ code: `${adapterType.split("_")[0]}_hello_probe_auth_required`, level: "warn", message: "Login required" }] });
|
||||
try {
|
||||
const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Login test", adapterType });
|
||||
const work = worker(createLifecycleDriver(db, {} as never).runHost);
|
||||
await work.process(agent.id);
|
||||
expect(probe).toHaveBeenCalled();
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." });
|
||||
} finally { probe.mockRestore(); }
|
||||
});
|
||||
|
||||
it("recovers an expired native owner only after the cancelled run and controller stop", async () => {
|
||||
const agent = await hire();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Cancelled native run" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, nativeIssueId: issue.id,
|
||||
status: "cancelled", invocationSource: "on_demand", runtimeMode: "native",
|
||||
startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning();
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id,
|
||||
phase: "executing", leaseOwner: "stopped-server", leaseExpiresAt: new Date(0), controllerPid: process.pid });
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
|
||||
await db.update(nativeRunFinalizations).set({ controllerPid: null }).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
|
||||
await db.update(heartbeatRuns).set({ processPid: 2_000_000_000 }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true);
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]).toMatchObject({ leaseOwner: null, leaseExpiresAt: null });
|
||||
});
|
||||
|
||||
it("does not advance termination while a cancelled process still runs", async () => {
|
||||
const agent = await hire();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id,
|
||||
status: "cancelled", invocationSource: "on_demand", runtimeMode: "legacy",
|
||||
startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning();
|
||||
const driver = createLifecycleDriver(db, {} as never);
|
||||
const work = worker(driver.runHost);
|
||||
await createAgentLifecycle(db).terminateAgent(agent.id);
|
||||
await work.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("terminating");
|
||||
await db.update(heartbeatRuns).set({ processPid: null,
|
||||
resultJson: { executionCancellation: { state: "acknowledged" } } }).where(eq(heartbeatRuns.id, run.id));
|
||||
await createAgentLifecycle(db).retry(agent.id);
|
||||
await work.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("terminated");
|
||||
});
|
||||
|
||||
it("does not cancel a reporting agent when its manager pauses", async () => {
|
||||
const parent = await hire();
|
||||
const [child] = await db.insert(agents).values({ companyId, name: "Report", reportsTo: parent.id,
|
||||
status: "idle", lifecycleState: "ready" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: child.id,
|
||||
status: "queued", invocationSource: "on_demand" }).returning();
|
||||
const work = worker(createLifecycleDriver(db, {} as never).runHost);
|
||||
await createAgentLifecycle(db).pauseAgent(parent.id);
|
||||
await work.process(parent.id);
|
||||
expect((await current(parent.id)).lifecycleState).toBe("paused");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]!.status).toBe("queued");
|
||||
});
|
||||
|
||||
it("retains company data until every agent finishes termination", async () => {
|
||||
const agent = await hire();
|
||||
const [project] = await db.insert(projects).values({ companyId, name: "Keep until cleanup", leadAgentId: agent.id }).returning();
|
||||
await db.insert(issues).values({ companyId, projectId: project.id, title: "Keep this task", assigneeAgentId: agent.id });
|
||||
const pending = worker(async () => "pending");
|
||||
await expect(deleteCompany(db, companyId)).rejects.toThrow("Complete termination");
|
||||
expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1);
|
||||
expect(await db.select().from(issues).where(eq(issues.companyId, companyId))).toHaveLength(1);
|
||||
await pending.stop();
|
||||
const complete = worker();
|
||||
await createAgentLifecycle(db).retry(agent.id);
|
||||
await complete.process(agent.id);
|
||||
await deleteCompany(db, companyId);
|
||||
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("requires a transaction for the company deletion integration", async () => {
|
||||
await expect(agentLifecycleCompanyDeletion.deleteCompanyData(db as never, companyId))
|
||||
.rejects.toThrow("Company deletion requires a database transaction");
|
||||
});
|
||||
|
||||
it("rejects individual deletion before cleanup and rolls back failed cleanup", async () => {
|
||||
const agent = await hire();
|
||||
const effects = createAgentLifecycleEffects();
|
||||
const cleanup = vi.fn(effects.deleteDependencies);
|
||||
const lifecycle = createLifecycleCommands(db, { ...effects, deleteDependencies: cleanup });
|
||||
await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Complete termination");
|
||||
expect(cleanup).not.toHaveBeenCalled();
|
||||
|
||||
await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, agent.id));
|
||||
const [key] = await db.insert(agentApiKeys).values({ companyId, agentId: agent.id, name: "Keep on rollback", keyHash: randomUUID() }).returning();
|
||||
cleanup.mockImplementation(async (tx, companyId, id) => {
|
||||
await effects.deleteDependencies(tx, companyId, id);
|
||||
throw new Error("Dependency cleanup failed");
|
||||
});
|
||||
await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Dependency cleanup failed");
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" });
|
||||
expect(await db.select().from(agentApiKeys).where(eq(agentApiKeys.id, key.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("deletes terminated and rejected agents only in the requested company", async () => {
|
||||
await db.insert(agents).values((["terminated", "rejected"] as const).map(lifecycleState => ({
|
||||
companyId, name: lifecycleState, status: "terminated", lifecycleState,
|
||||
})));
|
||||
const [otherCompany] = await db.insert(companies).values({ name: "Keep this company", issuePrefix: `K${companyId.slice(0, 7)}` }).returning();
|
||||
const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Keep this agent",
|
||||
status: "terminated", lifecycleState: "terminated" }).returning();
|
||||
|
||||
await deleteCompany(db, companyId);
|
||||
|
||||
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
|
||||
expect(await current(otherAgent.id)).toMatchObject({ companyId: otherCompany.id });
|
||||
expect(await db.select().from(companies).where(eq(companies.id, otherCompany.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rolls back agent and company data deletion when a later operation fails", async () => {
|
||||
const agent = await hire("terminated");
|
||||
const [project] = await db.insert(projects).values({ companyId, name: "Keep on rollback" }).returning();
|
||||
const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData;
|
||||
const failure = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => {
|
||||
await removeAgents(tx, id);
|
||||
expect(await tx.select().from(agents).where(eq(agents.companyId, id))).toHaveLength(0);
|
||||
throw new Error("Later deletion failed");
|
||||
});
|
||||
try {
|
||||
await expect(deleteCompany(db, companyId)).rejects.toThrow("Later deletion failed");
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" });
|
||||
expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1);
|
||||
expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(1);
|
||||
} finally { failure.mockRestore(); }
|
||||
});
|
||||
|
||||
it("blocks a concurrent hire until company deletion commits", async () => {
|
||||
let reportLock!: (pid: number) => void;
|
||||
let rejectLock!: (error: unknown) => void;
|
||||
const locked = new Promise<number>((resolve, reject) => { reportLock = resolve; rejectLock = reject; });
|
||||
let release!: () => void;
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData;
|
||||
const pause = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => {
|
||||
const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`);
|
||||
reportLock(backend!.pid);
|
||||
await released;
|
||||
await removeAgents(tx, id);
|
||||
});
|
||||
const deletion = deleteCompany(db, companyId);
|
||||
void deletion.catch(rejectLock);
|
||||
let hiring: Promise<unknown> | undefined;
|
||||
try {
|
||||
const pid = await locked;
|
||||
hiring = hire().then(() => "created", error => error.message);
|
||||
await vi.waitFor(async () => {
|
||||
const blocked = await db.execute(sql`select 1 from pg_stat_activity
|
||||
where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))`);
|
||||
expect(blocked).toHaveLength(1);
|
||||
});
|
||||
release();
|
||||
await deletion;
|
||||
expect(await hiring).toBe("Company not found");
|
||||
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
|
||||
} finally {
|
||||
release();
|
||||
await Promise.allSettled([deletion, hiring]);
|
||||
pause.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("requires cleanup proof after a remote run is already cancelled", async () => {
|
||||
const agent = await hire();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id,
|
||||
status: "cancelled", invocationSource: "on_demand", runtimeMode: "native",
|
||||
startedAt: new Date(), finishedAt: new Date() }).returning();
|
||||
const [lease] = await db.insert(environmentLeases).values({ companyId, heartbeatRunId: run.id,
|
||||
provider: "test", providerLeaseId: "test-lease", status: "pending_cleanup",
|
||||
cleanupStatus: "failed", releasedAt: new Date() }).returning();
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
|
||||
await db.update(environmentLeases).set({ metadata: {
|
||||
remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }),
|
||||
} }).where(eq(environmentLeases.id, lease.id));
|
||||
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true);
|
||||
});
|
||||
|
||||
it("does not prepare a proposed hire until approval", async () => {
|
||||
const observed: string[] = [];
|
||||
const work = worker(async agent => { observed.push(agent.lifecycleState); return "complete"; });
|
||||
const agent = await hire("pending_approval");
|
||||
await work.process(agent.id);
|
||||
expect(observed).toEqual([]);
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" });
|
||||
await approvalService(db).approve(approval.id, "board");
|
||||
await vi.waitFor(async () => expect((await current(agent.id)).lifecycleState).toBe("ready"));
|
||||
expect(observed).toEqual(["preparing", "verifying"]);
|
||||
});
|
||||
|
||||
it.each(["approved", "rejected"] as const)("uses the same %s hire decision for an agent and its approval", async decision => {
|
||||
const agent = await hire("pending_approval");
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
|
||||
requestedByUserId: "requester", payload: { agentId: agent.id }, status: "pending" });
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const decide = decision === "approved" ? lifecycle.approveHire : lifecycle.rejectHire;
|
||||
const first = await decide(agent.id, "board");
|
||||
expect(first).toMatchObject({ applied: true, approval: { id: approval.id, status: decision },
|
||||
agent: { lifecycleState: decision === "approved" ? "preparing" : "rejected" } });
|
||||
const version = (await current(agent.id)).lifecycleVersion;
|
||||
expect(await decide({ approvalId: approval.id }, "board")).toMatchObject({ applied: false, hireApprovedAgentId: null });
|
||||
expect((await current(agent.id)).lifecycleVersion).toBe(version);
|
||||
});
|
||||
|
||||
it("rolls back the approval and agent together when activation fails", async () => {
|
||||
const agent = await hire("pending_approval");
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
|
||||
payload: { agentId: agent.id }, status: "pending" });
|
||||
const lifecycle = createLifecycleCommands(db, { ...createAgentLifecycleEffects(),
|
||||
recordCreation: async () => { throw new Error("Creation event failed"); } });
|
||||
await expect(lifecycle.approveHire(agent.id, "board")).rejects.toThrow("Creation event failed");
|
||||
expect((await current(agent.id)).lifecycleState).toBe("pending_approval");
|
||||
expect((await db.select().from(approvals).where(eq(approvals.id, approval.id)))[0].status).toBe("pending");
|
||||
});
|
||||
|
||||
it("keeps the hire credential owner and recovers it for migrated approvals", async () => {
|
||||
for (const savedOwner of [null, "original-owner"]) {
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Migrated hire", status: "pending_approval",
|
||||
lifecycleState: "pending_approval", lifecycleOperation: savedOwner ? {
|
||||
id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, responsibleUserId: savedOwner,
|
||||
} : null }).returning();
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
|
||||
requestedByUserId: "requesting-member", payload: { agentId: agent.id }, status: "pending" });
|
||||
await approvalService(db).approve(approval.id, "approving-member");
|
||||
expect((await current(agent.id)).lifecycleOperation?.responsibleUserId).toBe(savedOwner ?? "requesting-member");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects transaction injection and generic state writes", async () => {
|
||||
const agent = await hire("pending_approval");
|
||||
await expect(db.transaction(async tx => createAgentLifecycle(tx as unknown as Db))).rejects.toThrow("root database");
|
||||
await expect(agentConfiguration(db).update(agent.id, { status: "idle" } as never)).rejects.toThrow("lifecycle command");
|
||||
await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Bypass", lifecycleRequiredPluginIds: [] } as never)).rejects.toThrow("Lifecycle fields");
|
||||
});
|
||||
|
||||
it("clears a rejected hire from the user's primary selection", async () => {
|
||||
const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Proposed", status: "pending_approval" }, { createdByUserId: "board" });
|
||||
const preference = () => db.select().from(userCompanyPreferences).where(eq(userCompanyPreferences.companyId, companyId));
|
||||
expect((await preference())[0]!.primaryAgentId).toBe(agent.id);
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" });
|
||||
await approvalService(db).reject(approval.id, "board");
|
||||
expect((await preference())[0]!.primaryAgentId).toBeNull();
|
||||
expect((await current(agent.id)).lifecycleState).toBe("rejected");
|
||||
});
|
||||
|
||||
it("rolls back configuration and its revision when the requested transition fails", async () => {
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const agent = await hire("terminated");
|
||||
await expect(lifecycle.updateAndTransition(agent.id, "resume", { name: "Must not persist" },
|
||||
{ recordRevision: { createdByUserId: "board", source: "patch" } })).rejects.toThrow("Cannot resume");
|
||||
expect((await current(agent.id)).name).toBe("Agent");
|
||||
expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toEqual([]);
|
||||
const active = await hire();
|
||||
expect(await lifecycle.updateAndTransition(active.id, "pause", { name: "Paused agent" })).toMatchObject({ name: "Paused agent", lifecycleState: "pausing" });
|
||||
});
|
||||
|
||||
it("preserves a disabled positive budget at approval and applies explicit configuration changes", async () => {
|
||||
const agent = await hire("pending_approval");
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(companyId, { scopeType: "agent", scopeId: agent.id, amount: 1000, isActive: false }, null);
|
||||
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
|
||||
payload: { agentId: agent.id, budgetMonthlyCents: 1000 }, status: "pending" });
|
||||
const policy = async () => (await db.select().from(budgetPolicies).where(eq(budgetPolicies.scopeId, agent.id)))[0]!;
|
||||
await approvalService(db).approve(approval.id, "board");
|
||||
expect(await policy()).toMatchObject({ amount: 1000, isActive: false });
|
||||
await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 2000 });
|
||||
expect(await policy()).toMatchObject({ amount: 2000, isActive: true });
|
||||
await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 0 });
|
||||
expect(await policy()).toMatchObject({ amount: 0, isActive: false });
|
||||
});
|
||||
|
||||
it.each(["updateConfiguration", "updateAndTransition"] as const)("limits %s budget reconciliation to the edited agent", async command => {
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const one = await hire();
|
||||
const two = await lifecycle.requestHire(companyId, { name: "Other budget target" });
|
||||
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId));
|
||||
if (command === "updateConfiguration") await lifecycle.updateConfiguration(one.id, { budgetMonthlyCents: 1000 });
|
||||
else await lifecycle.updateAndTransition(one.id, "pause", { budgetMonthlyCents: 1000 });
|
||||
expect((await current(one.id)).lifecycleHolds).toContain("company_paused");
|
||||
expect((await current(two.id)).lifecycleHolds).toEqual([]);
|
||||
});
|
||||
|
||||
it("limits invocation policy checks to the requested agent", async () => {
|
||||
const one = await hire();
|
||||
const two = await createAgentLifecycle(db).requestHire(companyId, { name: "Another agent" });
|
||||
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId));
|
||||
await budgetService(db).getInvocationBlock(companyId, one.id);
|
||||
expect((await current(one.id)).lifecycleHolds).toContain("company_paused");
|
||||
expect((await current(two.id)).lifecycleHolds).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps a manual hold when company policy changes", async () => {
|
||||
const work = worker(); const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id);
|
||||
await lifecycle.pauseAgent(agent.id); await work.process(agent.id);
|
||||
await db.update(companies).set({ status: "archived" }).where(eq(companies.id, companyId));
|
||||
await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id);
|
||||
await db.update(companies).set({ status: "active" }).where(eq(companies.id, companyId));
|
||||
await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id);
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "paused", lifecycleHolds: ["manual"] });
|
||||
await lifecycle.resumeAgent(agent.id); await work.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("ready");
|
||||
});
|
||||
|
||||
it("defers automatic policy changes while a task holds the agent lock", async () => {
|
||||
const agent = await hire("paused");
|
||||
await db.transaction(async tx => {
|
||||
await tx.select().from(agents).where(eq(agents.id, agent.id)).for("update");
|
||||
expect(await createLifecycleStore(db).change(agent.id, "reconcile")).toBeNull();
|
||||
});
|
||||
expect(await createLifecycleStore(db).change(agent.id, "reconcile")).not.toBeNull();
|
||||
});
|
||||
|
||||
it("retains the failed phase and the same operation for retry", async () => {
|
||||
let fail = true;
|
||||
const work = worker(async agent => { if (agent.lifecycleState === "verifying" && fail) throw new Error("private detail"); return "complete"; });
|
||||
const agent = await hire(); await work.process(agent.id);
|
||||
const failed = await current(agent.id);
|
||||
expect(failed.lifecycleState).toBe("verifying");
|
||||
expect(failed.lifecycleError).not.toContain("private detail");
|
||||
fail = false;
|
||||
await createAgentLifecycle(db).retry(agent.id); await work.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("ready");
|
||||
});
|
||||
|
||||
it("fences a late completion when termination replaces preparation", async () => {
|
||||
const agent = await hire("paused");
|
||||
const store = createLifecycleStore(db);
|
||||
await createAgentLifecycle(db).resumeAgent(agent.id);
|
||||
const claimed = await store.claim(agent.id, "old", new Date());
|
||||
expect(claimed).not.toBeNull();
|
||||
await store.setRequiredPlugins(claimed!, "old", []);
|
||||
await createAgentLifecycle(db).terminateAgent(agent.id);
|
||||
expect(await store.completeHost(claimed!, "old")).toBe(false);
|
||||
expect(await store.change(agent.id, "complete", { owner: "old", version: claimed!.lifecycleVersion })).toBeNull();
|
||||
expect((await current(agent.id)).lifecycleState).toBe("terminating");
|
||||
});
|
||||
|
||||
it("revokes keys at termination and waits for cleanup before deletion", async () => {
|
||||
let pending = true;
|
||||
const phases: string[] = [];
|
||||
const work = worker(async agent => { phases.push(agent.lifecycleState); return agent.lifecycleState === "cleaning_up" && pending ? "pending" : "complete"; });
|
||||
const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id);
|
||||
await db.insert(agentApiKeys).values({ agentId: agent.id, companyId, name: "Test", keyHash: randomUUID() });
|
||||
await lifecycle.terminateAgent(agent.id); await work.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("cleaning_up");
|
||||
expect((await db.select().from(agentApiKeys).where(eq(agentApiKeys.agentId, agent.id)))[0].revokedAt).not.toBeNull();
|
||||
await expect(agentConfiguration(db).remove(agent.id)).rejects.toThrow("Complete termination");
|
||||
pending = false; await lifecycle.retry(agent.id); await work.process(agent.id);
|
||||
expect(phases).toContain("terminating");
|
||||
expect((await current(agent.id)).lifecycleState).toBe("terminated");
|
||||
await agentConfiguration(db).remove(agent.id);
|
||||
expect(await current(agent.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains required plugin IDs and checks the operation and company on each reply", async () => {
|
||||
const agent = await hire("paused");
|
||||
const pluginId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "lifecycle-test", version: "1.0.0", status: "ready",
|
||||
manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage"] } as never });
|
||||
const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete" }));
|
||||
const driver = createLifecycleDriver(db, { call } as never);
|
||||
const snapshot = (await createLifecycleStore(db).get(agent.id))!;
|
||||
expect(await driver.requiredPluginIds(snapshot)).toContain(pluginId);
|
||||
await db.insert(pluginCompanySettings).values({ companyId, pluginId, enabled: false });
|
||||
expect(await driver.requiredPluginIds(snapshot)).not.toContain(pluginId);
|
||||
expect(await driver.requiredPluginIds({ ...snapshot, lifecycleRequiredPluginIds: [pluginId] })).toContain(pluginId);
|
||||
await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("unavailable");
|
||||
await db.update(pluginCompanySettings).set({ enabled: true }).where(eq(pluginCompanySettings.pluginId, pluginId));
|
||||
expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete");
|
||||
expect(call).toHaveBeenCalledWith(pluginId, "agentLifecycle", { companyId, agentId: agent.id,
|
||||
operationId: snapshot.lifecycleOperation!.id, version: snapshot.lifecycleVersion, phase: "paused" }, 30_000);
|
||||
call.mockResolvedValueOnce({ operationId: "stale", version: snapshot.lifecycleVersion, status: "complete" });
|
||||
await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("Invalid lifecycle result");
|
||||
});
|
||||
|
||||
it("keeps host completion and required plugin results across a worker restart", async () => {
|
||||
const pluginIds = [randomUUID(), randomUUID()];
|
||||
const runHost = vi.fn(async () => "complete" as const);
|
||||
const firstPlugin = vi.fn(async (_agent: LifecycleAgent, pluginId: string) =>
|
||||
pluginId === pluginIds[0] ? "complete" as const : "pending" as const);
|
||||
const first = configureAgentLifecycle(db, { requiredPluginIds: async () => pluginIds, runHost, runPlugin: firstPlugin });
|
||||
workers.push(first);
|
||||
const agent = await hire();
|
||||
await first.process(agent.id);
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "preparing", lifecycleRequiredPluginIds: pluginIds,
|
||||
lifecycleOperation: { hostComplete: true, completedPluginIds: [pluginIds[0]] } });
|
||||
await first.stop();
|
||||
|
||||
const discover = vi.fn(async () => []);
|
||||
const resumedPlugin = vi.fn(async (_agent: LifecycleAgent, _pluginId: string) => "complete" as const);
|
||||
const resumed = configureAgentLifecycle(db, { requiredPluginIds: discover, runHost, runPlugin: resumedPlugin });
|
||||
workers.push(resumed);
|
||||
await createAgentLifecycle(db).retry(agent.id);
|
||||
await resumed.process(agent.id);
|
||||
expect((await current(agent.id)).lifecycleState).toBe("ready");
|
||||
expect(discover).not.toHaveBeenCalled();
|
||||
expect(runHost).toHaveBeenCalledTimes(2);
|
||||
expect(resumedPlugin.mock.calls.map(call => call[1])).toEqual([pluginIds[1], ...pluginIds]);
|
||||
});
|
||||
|
||||
it("requires host and plugin completion and freezes the selected plugin IDs", async () => {
|
||||
const agent = await hire();
|
||||
const store = createLifecycleStore(db);
|
||||
const claimed = (await store.claim(agent.id, "test", new Date()))!;
|
||||
const pluginId = randomUUID();
|
||||
expect(await store.setRequiredPlugins(claimed, "test", [pluginId])).toBe(true);
|
||||
expect(await store.setRequiredPlugins(claimed, "test", [])).toBe(false);
|
||||
expect(await store.completePlugin(claimed, "test", randomUUID())).toBe(false);
|
||||
const complete = () => store.change(agent.id, "complete", { owner: "test", version: claimed.lifecycleVersion });
|
||||
expect(await complete()).toBeNull();
|
||||
expect(await store.completePlugin(claimed, "test", pluginId)).toBe(true);
|
||||
expect(await complete()).toBeNull();
|
||||
expect(await store.completeHost(claimed, "test")).toBe(true);
|
||||
expect(await complete()).toMatchObject({ lifecycleState: "verifying" });
|
||||
expect(await store.completePlugin(claimed, "test", pluginId)).toBe(false);
|
||||
});
|
||||
|
||||
it("fences verification when the saved configuration changes", async () => {
|
||||
const lifecycle = createAgentLifecycle(db); const store = createLifecycleStore(db);
|
||||
const agent = await hire();
|
||||
const claimed = (await store.claim(agent.id, "configuration-test", new Date()))!;
|
||||
await store.setRequiredPlugins(claimed, "configuration-test", []);
|
||||
await agentConfiguration(db).update(agent.id, { adapterConfig: { command: "echo" } });
|
||||
expect(await store.completeHost(claimed, "configuration-test")).toBe(false);
|
||||
expect((await lifecycle.get(agent.id))!.lifecycleVersion).toBeGreaterThan(claimed.lifecycleVersion);
|
||||
});
|
||||
|
||||
it("invalidates only changed execution configuration during setup", async () => {
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const agent = await hire();
|
||||
const before = await current(agent.id);
|
||||
await lifecycle.updateConfiguration(agent.id, { name: "Profile only" });
|
||||
await lifecycle.updateConfiguration(agent.id, { adapterConfig: before.adapterConfig });
|
||||
expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion);
|
||||
await lifecycle.updateConfiguration(agent.id, { runtimeConfig: { heartbeat: { enabled: false } } });
|
||||
expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion + 1);
|
||||
const work = worker();
|
||||
await work.process(agent.id);
|
||||
const ready = await current(agent.id);
|
||||
expect(ready.lifecycleState).toBe("ready");
|
||||
await lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } });
|
||||
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", lifecycleVersion: ready.lifecycleVersion });
|
||||
});
|
||||
|
||||
it("rejects identity, accounting, and unknown fields through every configuration entry point", async () => {
|
||||
const agent = await hire();
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const before = await current(agent.id);
|
||||
for (const patch of [{ id: randomUUID() }, { companyId: randomUUID() }, { spentMonthlyCents: 0 },
|
||||
{ spendMonthUtc: "2099-01" }, { lastHeartbeatAt: new Date() }, { errorReason: null },
|
||||
{ createdAt: new Date() }, { updatedAt: new Date() }, { lifecycleState: "ready" }, { futureColumn: true }]) {
|
||||
await expect(lifecycle.updateConfiguration(agent.id, patch as never)).rejects.toThrow("Unsupported configuration field");
|
||||
await expect(lifecycle.updateAndTransition(agent.id, "pause", patch as never)).rejects.toThrow("Unsupported configuration field");
|
||||
await expect(db.transaction(tx => updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, patch as never)))
|
||||
.rejects.toThrow("Unsupported configuration field");
|
||||
}
|
||||
expect(await current(agent.id)).toEqual(before);
|
||||
const effects = createAgentLifecycleEffects();
|
||||
const malformed = createLifecycleCommands(db, { ...effects, prepareConfiguration: async () => ({ id: randomUUID() }) as never });
|
||||
await expect(malformed.updateConfiguration(agent.id, { name: "Changed" })).rejects.toThrow("Unsupported configuration field");
|
||||
expect(await current(agent.id)).toEqual(before);
|
||||
});
|
||||
|
||||
it("records only the changed configuration fields for a legacy appearance", async () => {
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Legacy", appearance: null }).returning();
|
||||
await createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Renamed" },
|
||||
{ recordRevision: { source: "test", createdByUserId: "board" } });
|
||||
const revisions = await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id));
|
||||
expect(revisions).toHaveLength(1);
|
||||
expect(revisions[0].changedKeys).toEqual(["name"]);
|
||||
expect(revisions[0].beforeConfig.appearance).toEqual(revisions[0].afterConfig.appearance);
|
||||
});
|
||||
|
||||
it("accepts the unchanged resolved appearance of a pending hire", async () => {
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Pending legacy", status: "pending_approval", appearance: null }).returning();
|
||||
const resolved = await agentConfiguration(db).getById(agent.id);
|
||||
await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { appearance: resolved!.appearance }))
|
||||
.resolves.toMatchObject({ id: agent.id });
|
||||
await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Changed" }))
|
||||
.rejects.toThrow("Pending approval agent configuration cannot be changed");
|
||||
});
|
||||
|
||||
it("rolls back a root configuration command when dependent records fail", async () => {
|
||||
const agent = await hire();
|
||||
const before = await current(agent.id);
|
||||
const effects = createAgentLifecycleEffects();
|
||||
const lifecycle = createLifecycleCommands(db, { ...effects, completeConfiguration: async (...args) => {
|
||||
await effects.completeConfiguration(...args);
|
||||
throw new Error("Configuration completion failed");
|
||||
} });
|
||||
await expect(lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } },
|
||||
{ recordRevision: { source: "test", createdByUserId: "board" } })).rejects.toThrow("Configuration completion failed");
|
||||
expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig,
|
||||
lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation });
|
||||
expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rolls back configuration and verification invalidation in the binding transaction", async () => {
|
||||
const agent = await hire();
|
||||
const before = await current(agent.id);
|
||||
await expect(updateAgentConfigurationInTransaction(db, agent.id, { name: "Uncommitted" }))
|
||||
.rejects.toThrow("requires a database transaction");
|
||||
expect(() => updateAgentConfigurationInTransaction(db, agent.id, { budgetMonthlyCents: 100 } as never))
|
||||
.toThrow("Budget changes require a root lifecycle command");
|
||||
await expect(db.transaction(async tx => agentConfiguration(tx as unknown as Db).update(agent.id, { name: "Uncommitted" })))
|
||||
.rejects.toThrow("root database");
|
||||
await expect(db.transaction(async tx => {
|
||||
await updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, { adapterConfig: { command: "echo" } });
|
||||
expect((await tx.select().from(agents).where(eq(agents.id, agent.id)))[0].lifecycleVersion).toBe(before.lifecycleVersion + 1);
|
||||
throw new Error("Binding change failed");
|
||||
})).rejects.toThrow("Binding change failed");
|
||||
expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig,
|
||||
lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation });
|
||||
});
|
||||
|
||||
it("permits a board retry and rejects an agent or a different company", async () => {
|
||||
const agent = await hire();
|
||||
function app(actor: Express.Request["actor"]) {
|
||||
const app = express(); app.use(express.json());
|
||||
app.use((req, _res, next) => { req.actor = actor; next(); });
|
||||
app.use("/api", agentRoutes(db)); app.use(errorHandler); return app;
|
||||
}
|
||||
const board: Express.Request["actor"] = { type: "board", source: "session", userId: "board", companyIds: [companyId],
|
||||
memberships: [{ companyId, membershipRole: "admin", status: "active" }] };
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "admin" });
|
||||
await db.insert(principalPermissionGrants).values(["agents:create", "agents:configure"].map(permissionKey => ({ companyId, principalType: "user", principalId: "board", permissionKey })));
|
||||
const url = `/api/agents/${agent.id}/lifecycle/retry`;
|
||||
const response = await request(app(board)).post(url).send({});
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body.lifecycleState).toBe("preparing");
|
||||
expect(response.body).not.toHaveProperty("lifecycleOperation");
|
||||
expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, agent.id), eq(activityLog.action, "agent.lifecycle_retried"))))
|
||||
.toEqual([expect.objectContaining({ actorType: "user", actorId: "board" })]);
|
||||
await request(app({ type: "agent", source: "agent_key", companyId, agentId: agent.id })).post(url).send({}).expect(403);
|
||||
const denied = await request(app({ ...board, companyIds: [], memberships: [] })).post(url).send({});
|
||||
expect([403, 404]).toContain(denied.status);
|
||||
});
|
||||
|
||||
it("recovers an expired lease and never accepts an expired completion", async () => {
|
||||
const agent = await hire("paused"); const store = createLifecycleStore(db);
|
||||
await createAgentLifecycle(db).resumeAgent(agent.id);
|
||||
const claim = await store.claim(agent.id, "first", new Date(Date.now() - 180_000));
|
||||
expect(claim).not.toBeNull();
|
||||
await store.setRequiredPlugins(claim!, "first", []);
|
||||
await store.completeHost(claim!, "first");
|
||||
expect(await store.change(agent.id, "complete", { owner: "first", version: claim!.lifecycleVersion })).toBeNull();
|
||||
expect(await store.claim(agent.id, "second", new Date())).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("lifecycle transition rules", () => {
|
||||
it("does not bypass authorization, verification, or cleanup", () => {
|
||||
expect(() => transition("pending_approval", "resume")).toThrow();
|
||||
expect(transition("preparing", "complete")).toBe("verifying");
|
||||
expect(transition("terminating", "complete")).toBe("cleaning_up");
|
||||
expect(transition("cleaning_up", "complete")).toBe("terminated");
|
||||
expect(() => transition("terminated", "resume")).toThrow();
|
||||
expect(transition("terminating", "terminate")).toBe("terminating");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import { agents, companies, createDb, type Db } from "@paperclipai/db";
|
||||
import { asc, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { startAgentLifecycle } from "../index.js";
|
||||
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("lifecycle policy sweep", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-lifecycle-policy-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
it("caps each policy page, skips ineligible agents, and returns to the first page", async () => {
|
||||
const [company] = await db.insert(companies).values({ name: "Policy pages", issuePrefix: "POL" }).returning();
|
||||
const eligible = await db.insert(agents).values(Array.from({ length: 101 }, (_, index) => ({
|
||||
id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`,
|
||||
companyId: company.id, name: `Agent ${index}`, status: "idle", lifecycleState: "ready" as const,
|
||||
lifecycleHolds: ["company_paused"], pauseReason: "company_paused",
|
||||
}))).returning();
|
||||
const excluded = await db.insert(agents).values((["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"] as const).map(lifecycleState => ({
|
||||
companyId: company.id, name: lifecycleState, status: "terminated", lifecycleState,
|
||||
lifecycleHolds: ["company_paused"], pauseReason: "company_paused",
|
||||
}))).returning();
|
||||
const effects = createAgentLifecycleEffects();
|
||||
const transaction = vi.spyOn(effects, "transaction");
|
||||
const now = Date.now();
|
||||
const clock = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => ({ unref() {} }) as ReturnType<typeof setInterval>);
|
||||
const clearTimer = vi.spyOn(globalThis, "clearInterval").mockImplementation(() => {});
|
||||
const work = startAgentLifecycle(db, effects, {
|
||||
requiredPluginIds: async () => [], runHost: async () => "pending", runPlugin: async () => "pending",
|
||||
}, () => true);
|
||||
const rows = () => db.select().from(agents).where(eq(agents.companyId, company.id)).orderBy(asc(agents.id));
|
||||
try {
|
||||
await work.sweep();
|
||||
expect(transaction).toHaveBeenCalledTimes(100);
|
||||
expect((await rows()).filter(row => row.lifecycleHolds.length === 0)).toHaveLength(100);
|
||||
await work.sweep();
|
||||
expect(transaction).toHaveBeenCalledTimes(100);
|
||||
clock.mockReturnValue(now + 60_001);
|
||||
await work.sweep();
|
||||
expect(transaction).toHaveBeenCalledTimes(101);
|
||||
expect((await rows()).filter(row => row.lifecycleHolds.length > 0).map(row => row.id).sort())
|
||||
.toEqual(excluded.map(row => row.id).sort());
|
||||
await db.update(agents).set({ lifecycleHolds: ["company_paused"] }).where(eq(agents.id, eligible[0].id));
|
||||
clock.mockReturnValue(now + 120_002);
|
||||
await work.sweep();
|
||||
expect(transaction).toHaveBeenCalledTimes(201);
|
||||
expect((await rows())[0].lifecycleHolds).toEqual([]);
|
||||
} finally {
|
||||
await work.stop();
|
||||
transaction.mockRestore();
|
||||
clock.mockRestore();
|
||||
timer.mockRestore();
|
||||
clearTimer.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { agentAppearanceSchema } from "@paperclipai/shared";
|
||||
import { approvals, type Db } from "@paperclipai/db";
|
||||
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
||||
import { unprocessable } from "../../../errors.js";
|
||||
import { approvalResolutionRecords, resolvableApprovalStatuses } from "../../../lib/approval-records.js";
|
||||
import { agentRecords } from "./records.js";
|
||||
import { createLifecycleStore } from "./postgres.js";
|
||||
import type { LifecycleEffects } from "./effects.js";
|
||||
|
||||
export type HireDecisionTarget = string | { approvalId: string };
|
||||
type HireDecisionResult = {
|
||||
approval: typeof approvals.$inferSelect | null;
|
||||
agent: Awaited<ReturnType<ReturnType<typeof agentRecords>["getById"]>>;
|
||||
applied: boolean;
|
||||
activated: boolean;
|
||||
hireApprovedAgentId: string | null;
|
||||
};
|
||||
|
||||
export function hireApprovalService(db: Db, effects: LifecycleEffects) {
|
||||
const { getExistingApproval, resolveApproval } = approvalResolutionRecords(db);
|
||||
async function decide(target: HireDecisionTarget, status: "approved" | "rejected", userId: string, note?: string | null): Promise<HireDecisionResult | null> {
|
||||
const initial = typeof target === "string"
|
||||
? await agentRecords(db, effects).getById(target)
|
||||
: await getExistingApproval(target.approvalId);
|
||||
if (!initial) return null;
|
||||
if (typeof target !== "string" && (!("type" in initial) || initial.type !== "hire_agent")) throw unprocessable("Expected a hire approval");
|
||||
const companyId = initial.companyId;
|
||||
// Hire decisions acquire the company lock before changing an approval or agent.
|
||||
return effects.transaction(db, companyId, async (tx, publications) => {
|
||||
const records = agentRecords(tx, effects);
|
||||
let approvalId = typeof target === "string" ? null : target.approvalId;
|
||||
if (typeof target === "string") {
|
||||
const [open] = await tx.select({ id: approvals.id }).from(approvals).where(and(
|
||||
eq(approvals.companyId, companyId), eq(approvals.type, "hire_agent"),
|
||||
inArray(approvals.status, resolvableApprovalStatuses),
|
||||
sql`${approvals.payload}->>'agentId' = ${target}`,
|
||||
)).orderBy(desc(approvals.createdAt)).limit(1);
|
||||
approvalId = open?.id ?? null;
|
||||
}
|
||||
let approval: typeof approvals.$inferSelect | null = null;
|
||||
let applied = true;
|
||||
if (approvalId) {
|
||||
const result = await resolveApproval(approvalId, status, userId, note, tx);
|
||||
approval = result.approval;
|
||||
applied = result.applied;
|
||||
}
|
||||
const payload = approval?.payload ?? {};
|
||||
let agentId = typeof target === "string" ? target : typeof payload.agentId === "string" ? payload.agentId : null;
|
||||
if (applied && status === "approved") {
|
||||
if (agentId) {
|
||||
const activated = await records.activatePendingApproval(agentId, approval ? payload : null, approval ? approval.requestedByUserId : userId);
|
||||
if (!approval) applied = activated?.activated ?? false;
|
||||
} else {
|
||||
const created = await records.create(companyId, {
|
||||
name: String(payload.name ?? "New Agent"),
|
||||
appearance: payload.appearance == null ? undefined : agentAppearanceSchema.parse(payload.appearance),
|
||||
role: String(payload.role ?? "general"),
|
||||
title: typeof payload.title === "string" ? payload.title : null,
|
||||
reportsTo: typeof payload.reportsTo === "string" ? payload.reportsTo : null,
|
||||
capabilities: typeof payload.capabilities === "string" ? payload.capabilities : null,
|
||||
adapterType: String(payload.adapterType ?? "process"),
|
||||
adapterConfig:
|
||||
typeof payload.adapterConfig === "object" && payload.adapterConfig !== null
|
||||
? (payload.adapterConfig as Record<string, unknown>)
|
||||
: {},
|
||||
budgetMonthlyCents:
|
||||
typeof payload.budgetMonthlyCents === "number" ? payload.budgetMonthlyCents : 0,
|
||||
metadata:
|
||||
typeof payload.metadata === "object" && payload.metadata !== null
|
||||
? (payload.metadata as Record<string, unknown>)
|
||||
: null,
|
||||
status: "idle",
|
||||
spentMonthlyCents: 0,
|
||||
permissions: undefined,
|
||||
lastHeartbeatAt: null,
|
||||
}, { runnerResolved: true, createdByUserId: approval!.requestedByAgentId ? null : approval!.requestedByUserId, responsibleUserId: approval!.requestedByUserId });
|
||||
agentId = created.id;
|
||||
}
|
||||
if (approval && agentId && typeof payload.budgetMonthlyCents === "number" && payload.budgetMonthlyCents > 0) {
|
||||
await effects.setAgentBudget(tx, publications, companyId, agentId, payload.budgetMonthlyCents, userId);
|
||||
}
|
||||
} else if (applied && agentId) {
|
||||
if (approval) await records.rejectPendingHire(agentId);
|
||||
else {
|
||||
const before = await records.getById(agentId);
|
||||
await createLifecycleStore(tx, effects).change(agentId, "reject");
|
||||
applied = before?.lifecycleState !== "rejected";
|
||||
}
|
||||
}
|
||||
return { approval, agent: agentId ? await records.getById(agentId) : null, applied,
|
||||
activated: status === "approved" && applied,
|
||||
hireApprovedAgentId: status === "approved" && applied ? agentId : null };
|
||||
});
|
||||
}
|
||||
return { decide };
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { agents } from "@paperclipai/db";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { CompanyDeletionParticipant } from "../../../lib/company-deletion.js";
|
||||
import { AgentLifecycleConflict, assertAgentPurgeAllowed } from "../domain/policy.js";
|
||||
|
||||
export const agentLifecycleCompanyDeletion: CompanyDeletionParticipant = {
|
||||
async deleteCompanyData(tx, companyId) {
|
||||
if (!("nestedIndex" in tx)) {
|
||||
throw new AgentLifecycleConflict("Company deletion requires a database transaction");
|
||||
}
|
||||
const rows = await tx.select({ lifecycleState: agents.lifecycleState }).from(agents)
|
||||
.where(eq(agents.companyId, companyId)).for("update");
|
||||
for (const agent of rows) assertAgentPurgeAllowed(agent);
|
||||
await tx.delete(agents).where(and(
|
||||
eq(agents.companyId, companyId),
|
||||
inArray(agents.lifecycleState, ["terminated", "rejected"]),
|
||||
));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, type Db } from "@paperclipai/db";
|
||||
import { agentRecordQueries, AGENT_CONFIGURATION_FIELDS, jsonEqual, type AgentConfigurationPatch, type UpdateAgentOptions } from "../../../lib/agent-records.js";
|
||||
import type { ActivityPublication } from "../../../types/activity-publication.js";
|
||||
import { AgentLifecycleConflict } from "../domain/policy.js";
|
||||
import type { LifecycleEffects } from "./effects.js";
|
||||
|
||||
export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects, id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions, publications: ActivityPublication[] = []) {
|
||||
if (!("nestedIndex" in tx)) throw new AgentLifecycleConflict("Configuration integration requires a database transaction");
|
||||
assertConfigurationFields(data);
|
||||
const [current] = await tx.select().from(agents).where(eq(agents.id, id)).for("update");
|
||||
if (!current) return null;
|
||||
const patch = await effects.prepareConfiguration(tx, current, data, options);
|
||||
assertConfigurationFields(patch);
|
||||
const changed = ["adapterType", "adapterConfig", "runtimeConfig", "defaultEnvironmentId"].some(key =>
|
||||
Object.prototype.hasOwnProperty.call(patch, key) && !jsonEqual(patch[key as keyof typeof patch], current[key as keyof typeof current]));
|
||||
const invalidate = changed && ["preparing", "verifying", "resuming"].includes(current.lifecycleState);
|
||||
const [updated] = await tx.update(agents).set({ ...patch, updatedAt: new Date(), ...(invalidate ? {
|
||||
lifecycleVersion: current.lifecycleVersion + 1, lifecycleError: null,
|
||||
lifecycleOperation: { ...current.lifecycleOperation!, id: randomUUID(), hostComplete: false,
|
||||
completedPluginIds: [], leaseOwner: undefined, leaseUntil: undefined, retryAt: undefined },
|
||||
} : {}) }).where(eq(agents.id, id)).returning();
|
||||
await effects.completeConfiguration(tx, current, updated, patch, options, publications);
|
||||
return agentRecordQueries(tx).getById(id);
|
||||
}
|
||||
|
||||
function assertConfigurationFields(data: AgentConfigurationPatch) {
|
||||
if (Object.keys(data).some(field => !AGENT_CONFIGURATION_FIELDS.some(allowed => allowed === field))) {
|
||||
throw new AgentLifecycleConflict("Unsupported configuration field; use the command that owns this field, such as a lifecycle command");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { agents, type Db } from "@paperclipai/db";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { conflict } from "../../../errors.js";
|
||||
import { agentRecordQueries } from "../../../lib/agent-records.js";
|
||||
import { readBuiltInAgentMarker } from "../../../lib/built-in-agent-metadata.js";
|
||||
import { assertAgentPurgeAllowed } from "../domain/policy.js";
|
||||
import type { LifecycleEffects } from "./effects.js";
|
||||
|
||||
export async function purgeAgent(db: Db, effects: LifecycleEffects, id: string) {
|
||||
const { getById, normalizeAgentRow } = agentRecordQueries(db);
|
||||
const existing = await getById(id);
|
||||
if (!existing) return null;
|
||||
return effects.transaction(db, existing.companyId, async tx => {
|
||||
const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update");
|
||||
if (!agent) return null;
|
||||
const marker = readBuiltInAgentMarker(agent.metadata);
|
||||
if (marker) throw conflict("Built-in agents cannot be deleted; pause them instead", {
|
||||
code: "built_in_agent_undeletable", key: marker.key, featureKeys: marker.featureKeys,
|
||||
});
|
||||
assertAgentPurgeAllowed(agent);
|
||||
await effects.deleteDependencies(tx, agent.companyId, id);
|
||||
const [deleted] = await tx.delete(agents).where(and(
|
||||
eq(agents.id, id), inArray(agents.lifecycleState, ["terminated", "rejected"]),
|
||||
)).returning();
|
||||
return deleted ? normalizeAgentRow(deleted) : null;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { LifecycleFailure } from "../application/ports.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { budgetPolicies } from "@paperclipai/db";
|
||||
import type { CreateAgentData, CreateAgentOptions, AgentConfigurationPatch, AgentConfigurationRecord, AgentHireRecord, UpdateAgentOptions } from "../../../lib/agent-records.js";
|
||||
import type { ActivityPublication } from "../../../types/activity-publication.js";
|
||||
|
||||
/** Services supply integrations; lifecycle owns the transaction and state writes. */
|
||||
export interface LifecycleEffects {
|
||||
reportFailure(context: LifecycleFailure, error: unknown): void;
|
||||
deleteDependencies(db: Db, companyId: string, agentId: string): Promise<void>;
|
||||
prepareConfiguration(db: Db, current: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch>;
|
||||
completeConfiguration(db: Db, before: AgentConfigurationRecord, after: AgentConfigurationRecord, patch: AgentConfigurationPatch, options: UpdateAgentOptions | undefined, publications: ActivityPublication[]): Promise<void>;
|
||||
transaction<T>(db: Db, companyId: string, work: (tx: Db, publications: ActivityPublication[]) => Promise<T>): Promise<T>;
|
||||
policyBlocks(db: Db, policy: typeof budgetPolicies.$inferSelect): Promise<boolean>;
|
||||
setAgentBudget(db: Db, publications: ActivityPublication[], companyId: string, agentId: string, amount: number, userId: string | null, isActive?: boolean): Promise<unknown>;
|
||||
enforceBudget(db: Db, companyId: string, agentId: string): Promise<void>;
|
||||
recordCreation(db: Db, companyId: string, agentId: string): Promise<void>;
|
||||
recordStatus(db: Db, companyId: string, agentId: string, before: string, after: string): Promise<void>;
|
||||
clearPrimary(db: Db, companyId: string, agentId: string): Promise<unknown>;
|
||||
prepareHire(db: Db, companyId: string, data: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData>;
|
||||
initializeHire(db: Db, agent: AgentHireRecord, options?: CreateAgentOptions): Promise<void>;
|
||||
prepareHireApproval(db: Db, agent: AgentHireRecord, payload?: Record<string, unknown> | null): Promise<AgentConfigurationPatch>;
|
||||
completeHireApproval(db: Db, agent: AgentHireRecord, previous: AgentHireRecord): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, asc, eq, inArray, sql } from "drizzle-orm";
|
||||
import { agents, agentApiKeys, companies, budgetPolicies, type Db } from "@paperclipai/db";
|
||||
import type { AgentLifecycleOperation } from "@paperclipai/shared";
|
||||
import { AgentLifecycleConflict, compatibilityStatus, transition } from "../domain/policy.js";
|
||||
import type { LifecycleEffects } from "./effects.js";
|
||||
import type { LifecycleStore } from "../application/ports.js";
|
||||
|
||||
export function assertRootDatabase(db: Db) {
|
||||
if ("nestedIndex" in db) throw new AgentLifecycleConflict("Lifecycle commands require the root database connection");
|
||||
}
|
||||
|
||||
export function createLifecycleStore(db: Db, effects: LifecycleEffects): LifecycleStore {
|
||||
const get = (id: string) => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null);
|
||||
return {
|
||||
get,
|
||||
async change(id, command, options = {}) {
|
||||
const existing = await get(id);
|
||||
if (!existing) return null;
|
||||
return effects.transaction(db, existing.companyId, async tx => {
|
||||
const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update", command === "reconcile" ? { skipLocked: true } : undefined);
|
||||
if (!agent) return null;
|
||||
if (options.version !== undefined && options.version !== agent.lifecycleVersion) return null;
|
||||
if (command === "complete" && (!options.owner || !agent.lifecycleOperation?.leaseUntil || agent.lifecycleOperation?.leaseOwner !== options.owner ||
|
||||
Date.parse(agent.lifecycleOperation?.leaseUntil ?? "") <= Date.now() ||
|
||||
!agent.lifecycleOperation.hostComplete || agent.lifecycleRequiredPluginIds === null ||
|
||||
agent.lifecycleRequiredPluginIds.some(id => !agent.lifecycleOperation!.completedPluginIds.includes(id)))) return null;
|
||||
if (command === "retry") {
|
||||
transition(agent.lifecycleState, command);
|
||||
if (agent.lifecycleOperation?.leaseUntil && Date.parse(agent.lifecycleOperation.leaseUntil) > Date.now()) return agent;
|
||||
const [updated] = await tx.update(agents).set({ lifecycleError: null,
|
||||
lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, id)).returning();
|
||||
return updated;
|
||||
}
|
||||
if (command === "resume" && !["ready", "paused", "pausing", "resuming"].includes(agent.lifecycleState)) transition(agent.lifecycleState, command);
|
||||
let holds = agent.lifecycleHolds;
|
||||
const reason = options.reason ?? "manual";
|
||||
if (command === "reconcile") {
|
||||
if (["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"].includes(agent.lifecycleState)) return agent;
|
||||
const [company] = await tx.select().from(companies).where(eq(companies.id, agent.companyId));
|
||||
const policies = await tx.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, agent.companyId),
|
||||
eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, id)));
|
||||
let budgetBlocked = false;
|
||||
for (const policy of policies) if (await effects.policyBlocks(tx, policy)) budgetBlocked = true;
|
||||
holds = holds.filter(hold => hold !== "budget" && hold !== "company_archived" && hold !== "company_paused");
|
||||
if (budgetBlocked) holds.push("budget");
|
||||
if (company.status === "archived") holds.push("company_archived");
|
||||
if (company.status === "paused") holds.push("company_paused");
|
||||
}
|
||||
if (command === "pause") holds = [...new Set([...holds, reason])];
|
||||
if (command === "resume") holds = holds.filter(hold => reason === "user" ? ["budget", "company_archived", "company_paused"].includes(hold) : hold !== reason);
|
||||
let next = command === "reconcile"
|
||||
? holds.length && !["pausing", "paused"].includes(agent.lifecycleState) ? "pausing" as const
|
||||
: !holds.length && agent.lifecycleState === "paused" ? "resuming" as const : agent.lifecycleState
|
||||
: command === "resume" && (holds.length || agent.lifecycleState === "pausing") ? agent.lifecycleState
|
||||
: transition(agent.lifecycleState, command, agent.lifecycleOperation?.resumeState);
|
||||
if (command === "complete" && next === "ready" && holds.length) next = "pausing";
|
||||
if (command === "complete" && next === "paused" && !holds.length) next = "resuming";
|
||||
if (next === agent.lifecycleState) {
|
||||
if (JSON.stringify(holds) !== JSON.stringify(agent.lifecycleHolds) || agent.pauseReason !== (holds[0] ?? null)) {
|
||||
await tx.update(agents).set({ lifecycleHolds: holds, pauseReason: holds[0] ?? null }).where(eq(agents.id, id));
|
||||
}
|
||||
return agent;
|
||||
}
|
||||
const operation: AgentLifecycleOperation = {
|
||||
responsibleUserId: agent.lifecycleOperation?.responsibleUserId,
|
||||
id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0,
|
||||
resumeState: next === "pausing"
|
||||
? agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying" ? agent.lifecycleState : agent.lifecycleOperation?.resumeState ?? "ready"
|
||||
: agent.lifecycleOperation?.resumeState,
|
||||
};
|
||||
if (next === "ready") operation.resumeState = "ready";
|
||||
const status = compatibilityStatus(next, agent.status);
|
||||
const [updated] = await tx.update(agents).set({
|
||||
lifecycleHolds: holds, lifecycleState: next, lifecycleVersion: agent.lifecycleVersion + 1, lifecycleError: null,
|
||||
lifecycleOperation: operation, status,
|
||||
pauseReason: holds[0] ?? null,
|
||||
pausedAt: next === "pausing" ? new Date() : next === "ready" ? null : agent.pausedAt,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(agents.id, id)).returning();
|
||||
if (next === "terminating" || next === "rejected") {
|
||||
await tx.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id));
|
||||
await effects.clearPrimary(tx, agent.companyId, id);
|
||||
}
|
||||
if (command === "approve") await effects.recordCreation(tx, agent.companyId, id);
|
||||
if (next === "pausing" || next === "resuming" || next === "terminating" || next === "rejected") {
|
||||
await effects.recordStatus(tx, agent.companyId, id,
|
||||
next === "resuming" ? "paused" : "idle",
|
||||
next === "resuming" ? "idle" : next === "pausing" ? "paused" : "terminated");
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
},
|
||||
async claim(id, owner, now) {
|
||||
const [agent] = await db.update(agents).set({
|
||||
lifecycleOperation: sql`coalesce(${agents.lifecycleOperation}, '{"id":"", "hostComplete":false, "completedPluginIds":[], "attempts":0}'::jsonb)
|
||||
|| jsonb_build_object('leaseOwner', ${owner}::text, 'leaseUntil', ${(new Date(now.getTime() + 120_000)).toISOString()}::text,
|
||||
'attempts', coalesce((${agents.lifecycleOperation}->>'attempts')::integer, 0) + 1)`,
|
||||
}).where(and(eq(agents.id, id), inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]),
|
||||
sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
|
||||
sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
|
||||
)).returning();
|
||||
return agent ?? null;
|
||||
},
|
||||
async renew(agent, owner, now) {
|
||||
const rows = await db.update(agents).set({ lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{leaseUntil}', ${JSON.stringify(new Date(now.getTime() + 120_000).toISOString())}::jsonb)` })
|
||||
.where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
async setRequiredPlugins(agent, owner, pluginIds) {
|
||||
const rows = await db.update(agents).set({ lifecycleRequiredPluginIds: pluginIds })
|
||||
.where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
|
||||
sql`${agents.lifecycleRequiredPluginIds} is null`,
|
||||
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
async completeHost(agent, owner) {
|
||||
const rows = await db.update(agents).set({
|
||||
lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{hostComplete}', 'true'::jsonb)`,
|
||||
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
|
||||
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
async completePlugin(agent, owner, pluginId) {
|
||||
const rows = await db.update(agents).set({
|
||||
lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{completedPluginIds}',
|
||||
coalesce(${agents.lifecycleOperation}->'completedPluginIds', '[]'::jsonb) || ${JSON.stringify([pluginId])}::jsonb)`,
|
||||
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
|
||||
sql`${agents.lifecycleRequiredPluginIds} @> ${JSON.stringify([pluginId])}::jsonb`,
|
||||
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
async defer(agent, owner, error, now) {
|
||||
const rows = await db.update(agents).set({
|
||||
lifecycleError: error,
|
||||
lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil')
|
||||
|| jsonb_build_object('retryAt', ${new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString()}::text)`,
|
||||
updatedAt: now,
|
||||
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
|
||||
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
async pending(limit, now) {
|
||||
return db.select({ id: agents.id }).from(agents).where(and(
|
||||
inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]),
|
||||
sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
|
||||
sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
|
||||
)).orderBy(asc(agents.updatedAt), asc(agents.id)).limit(limit).then(rows => rows.map(row => row.id));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { LifecycleEffects } from "./effects.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, agentApiKeys, type Db } from "@paperclipai/db";
|
||||
import { conflict, notFound } from "../../../errors.js";
|
||||
import { agentRecordQueries, type CreateAgentOptions, type CreateAgentData } from "../../../lib/agent-records.js";
|
||||
|
||||
export function agentRecords(db: Db, effects: LifecycleEffects) {
|
||||
const { getById } = agentRecordQueries(db);
|
||||
|
||||
return {
|
||||
getById,
|
||||
create: async (companyId: string, data: CreateAgentData, options?: CreateAgentOptions) => {
|
||||
if (Object.keys(data).some(key => key.startsWith("lifecycle"))) throw conflict("Lifecycle fields belong to the lifecycle module");
|
||||
return effects.transaction(db, companyId, async (txDb) => {
|
||||
const tx = txDb;
|
||||
const prepared = await effects.prepareHire(txDb, companyId, data, options);
|
||||
const created = await tx
|
||||
.insert(agents)
|
||||
.values({
|
||||
...prepared,
|
||||
lifecycleState: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : data.status === "paused" ? "paused" : "preparing",
|
||||
lifecycleVersion: 1,
|
||||
lifecycleHolds: data.status === "paused" ? [data.pauseReason ?? "manual"] : [],
|
||||
pauseReason: data.status === "paused" ? data.pauseReason ?? "manual" : null,
|
||||
pausedAt: data.status === "paused" ? new Date() : null,
|
||||
lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, resumeState: "preparing", responsibleUserId: options?.responsibleUserId ?? options?.createdByUserId },
|
||||
status: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : "paused",
|
||||
companyId,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
await effects.initializeHire(txDb, created, options);
|
||||
if (created.status !== "pending_approval" && created.status !== "terminated") {
|
||||
await effects.recordCreation(txDb, companyId, created.id);
|
||||
}
|
||||
const normalizedCreated = await agentRecordQueries(txDb).getById(created.id);
|
||||
if (!normalizedCreated) {
|
||||
throw notFound("Agent not found");
|
||||
}
|
||||
return normalizedCreated;
|
||||
});
|
||||
},
|
||||
|
||||
clearError: async (id: string) => {
|
||||
const existing = await getById(id);
|
||||
if (!existing) return null;
|
||||
if (existing.status === "terminated") throw conflict("Cannot clear error on terminated agent");
|
||||
if (existing.status === "pending_approval") {
|
||||
throw conflict("Pending approval agents cannot have errors cleared");
|
||||
}
|
||||
if (existing.status !== "error") {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
.update(agents)
|
||||
.set({
|
||||
status: "idle",
|
||||
pauseReason: null,
|
||||
pausedAt: null,
|
||||
errorReason: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (!updated) {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
return getById(updated.id);
|
||||
},
|
||||
|
||||
rejectPendingHire: async (id: string) => {
|
||||
const rows = await db.update(agents).set({ status: "terminated", lifecycleState: "rejected",
|
||||
lifecycleVersion: sql`${agents.lifecycleVersion} + 1`, lifecycleOperation: null,
|
||||
updatedAt: new Date() }).where(and(eq(agents.id, id), eq(agents.lifecycleState, "pending_approval"))).returning();
|
||||
for (const row of rows) {
|
||||
await effects.clearPrimary(db, row.companyId, id);
|
||||
await db.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id));
|
||||
await effects.recordStatus(db, row.companyId, id, "pending_approval", "terminated");
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
|
||||
activatePendingApproval: async (id: string, approvedPayload?: Record<string, unknown> | null, requestedByUserId?: string | null) => {
|
||||
const activatedAgent = await db.transaction(async (tx) => {
|
||||
const txDb = tx as unknown as Db;
|
||||
const existing = await agentRecordQueries(txDb).getById(id);
|
||||
if (!existing || existing.status !== "pending_approval") return null;
|
||||
const patch = await effects.prepareHireApproval(txDb, existing, approvedPayload);
|
||||
const updated = await tx
|
||||
.update(agents)
|
||||
.set({ ...patch, status: "paused", lifecycleState: "preparing", lifecycleVersion: existing.lifecycleVersion + 1, lifecycleError: null, lifecycleOperation: sql`jsonb_build_object('id', ${randomUUID()}::text, 'hostComplete', false, 'completedPluginIds', '[]'::jsonb, 'attempts', 0, 'responsibleUserId', coalesce(nullif(${agents.lifecycleOperation}->'responsibleUserId', 'null'::jsonb), to_jsonb(${requestedByUserId ?? null}::text)))`, updatedAt: new Date() })
|
||||
.where(and(eq(agents.id, id), eq(agents.status, "pending_approval")))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!updated) return null;
|
||||
await effects.completeHireApproval(txDb, updated, existing);
|
||||
await effects.recordCreation(txDb, existing.companyId, updated.id);
|
||||
const agent = await agentRecordQueries(txDb).getById(updated.id);
|
||||
if (!agent) {
|
||||
throw notFound("Agent not found");
|
||||
}
|
||||
return agent;
|
||||
});
|
||||
|
||||
if (activatedAgent) {
|
||||
return { agent: activatedAgent, activated: true };
|
||||
}
|
||||
|
||||
const existing = await getById(id);
|
||||
return existing ? { agent: existing, activated: false } : null;
|
||||
},
|
||||
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { AgentLifecycleOperation, AgentLifecycleState } from "@paperclipai/shared";
|
||||
import type { LifecycleCommand } from "../domain/policy.js";
|
||||
|
||||
export interface LifecycleAgent {
|
||||
id: string;
|
||||
companyId: string;
|
||||
lifecycleState: AgentLifecycleState;
|
||||
lifecycleVersion: number;
|
||||
lifecycleRequiredPluginIds: string[] | null;
|
||||
lifecycleError: string | null;
|
||||
lifecycleOperation: AgentLifecycleOperation | null;
|
||||
}
|
||||
|
||||
export interface LifecycleStore {
|
||||
get(id: string): Promise<LifecycleAgent | null>;
|
||||
change(id: string, command: LifecycleCommand, options?: { version?: number; owner?: string; reason?: string }): Promise<LifecycleAgent | null>;
|
||||
claim(id: string, owner: string, now: Date): Promise<LifecycleAgent | null>;
|
||||
renew(agent: LifecycleAgent, owner: string, now: Date): Promise<boolean>;
|
||||
setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise<boolean>;
|
||||
completeHost(agent: LifecycleAgent, owner: string): Promise<boolean>;
|
||||
completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise<boolean>;
|
||||
defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date): Promise<boolean>;
|
||||
pending(limit: number, now: Date): Promise<string[]>;
|
||||
}
|
||||
|
||||
export interface LifecycleDriver {
|
||||
requiredPluginIds(agent: LifecycleAgent): Promise<string[]>;
|
||||
runHost(agent: LifecycleAgent): Promise<"complete" | "pending">;
|
||||
runPlugin(agent: LifecycleAgent, pluginId: string): Promise<"complete" | "pending">;
|
||||
}
|
||||
|
||||
export interface LifecycleFailure {
|
||||
stage: "claim" | "renew" | "select_plugins" | "save_plugins" | "host" | "plugin"
|
||||
| "complete_host" | "complete_plugin" | "transition" | "defer" | "scan" | "policy_scan";
|
||||
agentId?: string;
|
||||
companyId?: string;
|
||||
phase?: AgentLifecycleState;
|
||||
operationId?: string;
|
||||
version?: number;
|
||||
pluginId?: string;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js";
|
||||
import { isLifecycleWorkPending } from "../domain/policy.js";
|
||||
|
||||
/** Database leases protect each attempt across server processes. */
|
||||
export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) {
|
||||
let stopped = false;
|
||||
const active = new Map<string, Promise<void>>();
|
||||
let running: Promise<void> | undefined;
|
||||
|
||||
async function attempt<T>(context: LifecycleFailure, work: () => Promise<T>): Promise<T> {
|
||||
try { return await work(); }
|
||||
catch (error) { reportFailure(context, error); throw error; }
|
||||
}
|
||||
|
||||
async function process(id: string) {
|
||||
phase: for (let step = 0; !stopped && canRun() && step < 8; step++) {
|
||||
const owner = randomUUID();
|
||||
const agent = await attempt({ stage: "claim", agentId: id }, () => store.claim(id, owner, new Date()));
|
||||
if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return;
|
||||
const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState,
|
||||
operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion };
|
||||
const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) =>
|
||||
attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work);
|
||||
const renewal = setInterval(() => {
|
||||
void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {});
|
||||
}, 30_000);
|
||||
renewal.unref?.();
|
||||
try {
|
||||
const operation = agent.lifecycleOperation!;
|
||||
const pluginIds = agent.lifecycleRequiredPluginIds ?? await runStep("select_plugins", () => driver.requiredPluginIds(agent));
|
||||
if (agent.lifecycleRequiredPluginIds === null && !await runStep("save_plugins", () => store.setRequiredPlugins(agent, owner, pluginIds))) continue;
|
||||
if (!canRun()) {
|
||||
await runStep("defer", () => store.defer(agent, owner, null, new Date()));
|
||||
return;
|
||||
}
|
||||
if (!operation.hostComplete) {
|
||||
if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue;
|
||||
if (await runStep("host", () => driver.runHost(agent)) === "pending") {
|
||||
if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue;
|
||||
return;
|
||||
}
|
||||
if (!await runStep("complete_host", () => store.completeHost(agent, owner))) continue;
|
||||
}
|
||||
for (const pluginId of pluginIds) {
|
||||
if (operation.completedPluginIds.includes(pluginId)) continue;
|
||||
if (!canRun()) {
|
||||
await runStep("defer", () => store.defer(agent, owner, null, new Date()));
|
||||
return;
|
||||
}
|
||||
if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue phase;
|
||||
if (await runStep("plugin", () => driver.runPlugin(agent, pluginId), pluginId) === "pending") {
|
||||
if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue phase;
|
||||
return;
|
||||
}
|
||||
if (!await runStep("complete_plugin", () => store.completePlugin(agent, owner, pluginId), pluginId)) continue phase;
|
||||
}
|
||||
const changed = await runStep("transition", () => store.change(id, "complete", { version: agent.lifecycleVersion, owner }));
|
||||
if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return;
|
||||
} catch {
|
||||
if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue;
|
||||
return;
|
||||
} finally {
|
||||
clearInterval(renewal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function sweep() {
|
||||
const pending = await attempt({ stage: "scan" }, () => store.pending(100, new Date()));
|
||||
const entries = pending.values();
|
||||
await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => {
|
||||
for (const id of entries) await start(id);
|
||||
}));
|
||||
}
|
||||
|
||||
function start(id: string): Promise<void> {
|
||||
if (stopped || !canRun()) return Promise.resolve();
|
||||
const existing = active.get(id);
|
||||
if (existing) return existing;
|
||||
const work = process(id).finally(() => { active.delete(id); });
|
||||
active.set(id, work);
|
||||
return work;
|
||||
}
|
||||
|
||||
return {
|
||||
process: start,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
await Promise.allSettled([...active.values()]);
|
||||
},
|
||||
sweep() {
|
||||
return running ??= sweep().finally(() => { running = undefined; });
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { agentLifecycleCompanyDeletion } from "./adapters/company-deletion.js";
|
||||
@@ -0,0 +1 @@
|
||||
export { updateAgentConfiguration } from "./adapters/configuration.js";
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { AgentLifecycleState } from "@paperclipai/shared";
|
||||
|
||||
export type LifecycleCommand = "reconcile" | "approve" | "reject" | "pause" | "resume" | "terminate" | "complete" | "retry";
|
||||
export class AgentLifecycleConflict extends Error {
|
||||
constructor(message: string) { super(message); this.name = "AgentLifecycleConflict"; }
|
||||
}
|
||||
|
||||
export function canConfigureAgentConnection(agent: { lifecycleState: string; status: string }): boolean {
|
||||
return agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying"
|
||||
|| (agent.lifecycleState === "ready" && !["paused", "terminated", "pending_approval"].includes(agent.status));
|
||||
}
|
||||
|
||||
export function isAgentAwaitingSetup(agent: { lifecycleState: string; lifecycleHolds: readonly string[] }): boolean {
|
||||
return ["preparing", "verifying", "resuming"].includes(agent.lifecycleState) && agent.lifecycleHolds.length === 0;
|
||||
}
|
||||
|
||||
export function transition(state: AgentLifecycleState, command: LifecycleCommand,
|
||||
resumeState: "preparing" | "verifying" | "ready" = "ready"): AgentLifecycleState {
|
||||
if (command === "retry") {
|
||||
if (["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state)) return state;
|
||||
} else if (command === "terminate") {
|
||||
if (["terminated", "rejected", "terminating", "cleaning_up"].includes(state)) return state;
|
||||
return "terminating";
|
||||
} else if (command === "approve") {
|
||||
if (state === "pending_approval") return "preparing";
|
||||
} else if (command === "reject") {
|
||||
if (state === "pending_approval") return "rejected";
|
||||
if (state === "rejected") return state;
|
||||
} else if (command === "pause") {
|
||||
if (state === "paused" || state === "pausing") return state;
|
||||
if (["ready", "preparing", "verifying", "resuming"].includes(state)) return "pausing";
|
||||
} else if (command === "resume") {
|
||||
if (state === "ready" || state === "resuming") return state;
|
||||
if (state === "paused") return "resuming";
|
||||
} else if (command === "complete") {
|
||||
switch (state) {
|
||||
case "preparing": return "verifying";
|
||||
case "verifying": return "ready";
|
||||
case "pausing": return "paused";
|
||||
case "resuming": return resumeState;
|
||||
case "terminating": return "cleaning_up";
|
||||
case "cleaning_up": return "terminated";
|
||||
}
|
||||
}
|
||||
throw new AgentLifecycleConflict(`Cannot ${command} an agent in state ${state}`);
|
||||
}
|
||||
|
||||
export function compatibilityStatus(state: AgentLifecycleState, executionStatus = "idle"): string {
|
||||
if (state === "ready") return ["active", "idle", "running", "error"].includes(executionStatus) ? executionStatus : "idle";
|
||||
if (state === "pending_approval") return "pending_approval";
|
||||
if (["terminating", "cleaning_up", "terminated", "rejected"].includes(state)) return "terminated";
|
||||
return "paused";
|
||||
}
|
||||
|
||||
export function isLifecycleWorkPending(state: AgentLifecycleState): boolean {
|
||||
return ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state);
|
||||
}
|
||||
|
||||
export function assertAgentPurgeAllowed(agent: { lifecycleState: string }) {
|
||||
if (!["terminated", "rejected"].includes(agent.lifecycleState)) {
|
||||
throw new AgentLifecycleConflict("Complete termination before deleting the agent");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { hireApprovalService as approvalRecords, type HireDecisionTarget } from "./adapters/approvals.js";
|
||||
import { agents } from "@paperclipai/db";
|
||||
import { and, asc, eq, gt, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { LifecycleEffects } from "./adapters/effects.js";
|
||||
export type { LifecycleEffects } from "./adapters/effects.js";
|
||||
import { updateAgentConfiguration } from "./adapters/configuration.js";
|
||||
import type { AgentConfigurationPatch, UpdateAgentOptions } from "../../lib/agent-records.js";
|
||||
import { purgeAgent } from "./adapters/deletion.js";
|
||||
import { agentRecords } from "./adapters/records.js";
|
||||
import { assertRootDatabase, createLifecycleStore } from "./adapters/postgres.js";
|
||||
import { createLifecycleWorker } from "./application/worker.js";
|
||||
import type { LifecycleDriver } from "./application/ports.js";
|
||||
|
||||
export { AgentLifecycleConflict, canConfigureAgentConnection, isAgentAwaitingSetup } from "./domain/policy.js";
|
||||
|
||||
export type { HireDecisionTarget } from "./adapters/approvals.js";
|
||||
export type { LifecycleDriver, LifecycleAgent, LifecycleFailure } from "./application/ports.js";
|
||||
|
||||
const policyStates = ["preparing", "verifying", "ready", "pausing", "paused", "resuming"] as const;
|
||||
const policyPageSize = 100;
|
||||
|
||||
const workers = new WeakMap<Db, ReturnType<typeof createLifecycleWorker>>();
|
||||
export function configureAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun = () => true) {
|
||||
assertRootDatabase(db);
|
||||
const worker = createLifecycleWorker(createLifecycleStore(db, effects), driver, effects.reportFailure, canRun);
|
||||
workers.set(db, worker);
|
||||
return worker;
|
||||
}
|
||||
export function scheduleAgentLifecycle(db: Db, id: string) {
|
||||
// The periodic sweep retries work if the process stops before this call.
|
||||
return workers.get(db)?.process(id).catch(() => {});
|
||||
}
|
||||
|
||||
export function createAgentLifecycle(db: Db, effects: LifecycleEffects) {
|
||||
assertRootDatabase(db);
|
||||
const records = agentRecords(db, effects);
|
||||
const store = createLifecycleStore(db, effects);
|
||||
async function change(id: string, command: "pause" | "resume" | "terminate" | "retry", reason?: string) {
|
||||
const result = await store.change(id, command, { reason });
|
||||
if (result) scheduleAgentLifecycle(db, id);
|
||||
return result ? records.getById(id) : null;
|
||||
}
|
||||
async function decideHire(target: HireDecisionTarget, decision: "approved" | "rejected", userId: string, note?: string | null) {
|
||||
const result = await approvalRecords(db, effects).decide(target, decision, userId, note);
|
||||
if (result?.hireApprovedAgentId) scheduleAgentLifecycle(db, result.hireApprovedAgentId);
|
||||
return result;
|
||||
}
|
||||
return {
|
||||
async get(id: string) {
|
||||
const agent = await store.get(id);
|
||||
return agent ? {
|
||||
id: agent.id, companyId: agent.companyId, lifecycleState: agent.lifecycleState,
|
||||
lifecycleVersion: agent.lifecycleVersion, lifecycleError: agent.lifecycleError,
|
||||
} : null;
|
||||
},
|
||||
async requestHire(...args: Parameters<typeof records.create>) {
|
||||
const agent = await records.create(...args);
|
||||
scheduleAgentLifecycle(db, agent.id);
|
||||
return agent;
|
||||
},
|
||||
approveHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "approved", userId, note),
|
||||
rejectHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "rejected", userId, note),
|
||||
async updateConfiguration(id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions) {
|
||||
const existing = await records.getById(id);
|
||||
if (!existing) return null;
|
||||
const agent = await effects.transaction(db, existing.companyId, (tx, publications) =>
|
||||
updateAgentConfiguration(tx, effects, id, data, options, publications));
|
||||
if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id);
|
||||
scheduleAgentLifecycle(db, id);
|
||||
return agent;
|
||||
},
|
||||
async updateAndTransition(id: string, command: "pause" | "resume" | "terminate", data: AgentConfigurationPatch, options?: UpdateAgentOptions) {
|
||||
const existing = await records.getById(id);
|
||||
if (!existing) return null;
|
||||
const agent = await effects.transaction(db, existing.companyId, async (tx, publications) => {
|
||||
await updateAgentConfiguration(tx, effects, id, data, options, publications);
|
||||
await createLifecycleStore(tx, effects).change(id, command, { reason: command === "resume" ? "user" : "manual" });
|
||||
return agentRecords(tx, effects).getById(id);
|
||||
});
|
||||
if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id);
|
||||
scheduleAgentLifecycle(db, id);
|
||||
return agent;
|
||||
},
|
||||
async reconcilePolicyHolds(companyId: string, agentId?: string | null) {
|
||||
if (agentId === null) return;
|
||||
const rows = await db.select({ id: agents.id }).from(agents).where(and(
|
||||
eq(agents.companyId, companyId), agentId ? eq(agents.id, agentId) : undefined,
|
||||
inArray(agents.lifecycleState, policyStates),
|
||||
));
|
||||
for (const row of rows) {
|
||||
await store.change(row.id, "reconcile");
|
||||
scheduleAgentLifecycle(db, row.id);
|
||||
}
|
||||
},
|
||||
pauseAgent: (id: string, reason = "manual") => change(id, "pause", reason),
|
||||
resumeAgent: (id: string, reason = "user") => change(id, "resume", reason),
|
||||
terminateAgent: (id: string) => change(id, "terminate"),
|
||||
retry: (id: string) => change(id, "retry"),
|
||||
clearError: records.clearError,
|
||||
purgeAgent: (id: string) => purgeAgent(db, effects, id),
|
||||
};
|
||||
}
|
||||
|
||||
export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun: () => boolean) {
|
||||
const worker = configureAgentLifecycle(db, effects, driver, canRun);
|
||||
let running: Promise<void> | undefined;
|
||||
let stopped = false;
|
||||
let nextPolicySweep = 0;
|
||||
let policyCursor: string | undefined;
|
||||
const store = createLifecycleStore(db, effects);
|
||||
function sweep() {
|
||||
if (stopped || !canRun()) return Promise.resolve();
|
||||
return running ??= (async () => {
|
||||
if (Date.now() >= nextPolicySweep) {
|
||||
try {
|
||||
const rows = await db.select({ id: agents.id }).from(agents).where(and(
|
||||
inArray(agents.lifecycleState, policyStates), policyCursor ? gt(agents.id, policyCursor) : undefined,
|
||||
)).orderBy(asc(agents.id)).limit(policyPageSize);
|
||||
for (const row of rows) {
|
||||
await store.change(row.id, "reconcile");
|
||||
scheduleAgentLifecycle(db, row.id);
|
||||
policyCursor = row.id;
|
||||
}
|
||||
if (rows.length < policyPageSize) policyCursor = undefined;
|
||||
}
|
||||
catch (error) { effects.reportFailure({ stage: "policy_scan" }, error); throw error; }
|
||||
nextPolicySweep = Date.now() + 60_000;
|
||||
}
|
||||
await worker.sweep();
|
||||
})().finally(() => { running = undefined; });
|
||||
}
|
||||
const timer = setInterval(() => { void sweep().catch(() => {}); }, 5_000);
|
||||
timer.unref?.();
|
||||
return {
|
||||
sweep,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
workers.delete(db);
|
||||
try { await running; } finally { await worker.stop(); }
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user