merge: integrate agent lifecycle ownership from master

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-09 11:24:59 -05:00
commit ad5d8826ad
159 files changed
+7173 -2805

No files matched your search

+30 -3
View File
@@ -22,10 +22,37 @@ adapters → application → domain
Postgres queries, transactions, and process control. An adapter file may
import `drizzle-orm`, `@paperclipai/db`, and Node.js I/O modules.
A module exposes one entry point, `index.ts`, which composes the adapters
A module exposes its commands through `index.ts`, which composes the adapters
and the use cases behind a factory function. Code outside the module imports
only that entry point, never a file inside `domain/`, `application/`, or
that entry point, never a file inside `domain/`, `application/`, or
`adapters/` directly.
`pnpm check:module-boundaries` enforces these rules for production source
files. It also rejects imports that bypass another module's `index.ts`.
files. It also checks access to the company deletion entry point below.
## Company deletion
The existing company deletion service owns the database transaction and deletion
order. A module can expose a separate `company-deletion.ts` entry point.
Only `services/company-deletion.ts` can import this entry point.
The entry point exports an implementation of `CompanyDeletionParticipant` from
`lib/company-deletion.ts`. Do not export it through the normal module index.
The service must complete external cleanup before it starts the transaction.
It must lock the company before it deletes dependent records or calls a module.
Creation commands must use the same company lock.
The service removes dependent records before it calls the module.
The module checks its deletion conditions and deletes the records it owns.
Use the supplied transaction. Do not start another transaction or call external
systems. Throw an error if deletion cannot proceed.
An error must roll back all database changes in the deletion transaction.
Call each module explicitly in dependency order. No module registry is required.
The agent lifecycle module also has a restricted `configuration.ts` entry point.
It saves configuration and invalidates verification in the caller's transaction.
The [lifecycle guide](../../../doc/AGENT-LIFECYCLE.md) lists its permitted callers.
Ordinary configuration updates use the root lifecycle command.
The [agent lifecycle module](../../../doc/AGENT-LIFECYCLE.md) owns agent creation,
pause, resume, and termination. Use its commands for these changes.
@@ -456,6 +456,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
eq(agents.id, input.run.agentId),
eq(agents.companyId, companyId),
notInArray(agents.status, ["paused", "terminated"]),
eq(agents.lifecycleState, "ready"),
));
return updatedRun;
@@ -0,0 +1,68 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { createLifecycleWorker } from "../application/worker.js";
import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../application/ports.js";
import { startAgentLifecycle } from "../index.js";
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
import { logger } from "../../../middleware/logger.js";
const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3,
lifecycleRequiredPluginIds: null, lifecycleError: null,
lifecycleOperation: { id: "operation", hostComplete: false, completedPluginIds: [], attempts: 1 } };
function fixture() {
const store = { get: vi.fn(), claim: vi.fn().mockResolvedValue(agent), renew: vi.fn().mockResolvedValue(true),
setRequiredPlugins: vi.fn().mockResolvedValue(true), completeHost: vi.fn().mockResolvedValue(true),
completePlugin: vi.fn().mockResolvedValue(true), defer: vi.fn().mockResolvedValue(true),
change: vi.fn().mockResolvedValue({ ...agent, lifecycleState: "ready" }), pending: vi.fn().mockResolvedValue([agent.id]) } satisfies LifecycleStore;
const driver = { requiredPluginIds: vi.fn().mockResolvedValue(["plugin"]), runHost: vi.fn().mockResolvedValue("complete"),
runPlugin: vi.fn().mockResolvedValue("complete") } satisfies LifecycleDriver;
const report = vi.fn();
return { store, driver, report, worker: createLifecycleWorker(store, driver, report) };
}
afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
describe("lifecycle failure diagnostics", () => {
it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => {
const { store, driver, report, worker } = fixture();
const error = new Error("private provider detail");
if (stage === "host") driver.runHost.mockRejectedValue(error);
else if (stage === "plugin") driver.runPlugin.mockRejectedValue(error);
else if (stage === "scan") store.pending.mockRejectedValue(error);
else if (stage === "defer") { driver.runHost.mockResolvedValue("pending"); store.defer.mockRejectedValue(error); }
else store[stage].mockRejectedValue(error);
const work = stage === "scan" ? worker.sweep() : worker.process(agent.id);
if (["claim", "scan", "defer"].includes(stage)) await expect(work).rejects.toThrow(error);
else await work;
expect(report).toHaveBeenCalledWith(expect.objectContaining({ stage }), error);
if (!["claim", "scan"].includes(stage)) expect(report).toHaveBeenCalledWith(expect.objectContaining({
agentId: agent.id, companyId: agent.companyId, phase: "verifying", operationId: "operation", version: 3,
...(stage === "plugin" ? { pluginId: "plugin" } : {}),
}), error);
await worker.stop();
});
it("reports periodic policy scan failures", async () => {
vi.useFakeTimers();
const error = new Error("private database detail");
const db = { select: () => { throw error; } } as unknown as Db;
const effects = { ...createAgentLifecycleEffects(), reportFailure: vi.fn() };
const work = startAgentLifecycle(db, effects, fixture().driver, () => true);
try {
await vi.advanceTimersByTimeAsync(5_000);
expect(effects.reportFailure).toHaveBeenCalledWith({ stage: "policy_scan" }, error);
} finally { await work.stop(); }
});
it.each([
[{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"],
[{ code: "harness_test_failed" }, "harness_test_failed"],
[{ cause: { code: "40P01", message: "private query" } }, "40P01"],
[{ code: "private credential value" }, "unclassified"],
])("logs known codes without provider error contents", (details, failureCode) => {
const log = vi.spyOn(logger, "warn").mockImplementation(() => {});
const context = { stage: "host" as const, agentId: "agent", phase: "verifying" as const, operationId: "operation", version: 3 };
createAgentLifecycleEffects().reportFailure(context, Object.assign(new Error("private provider detail"), details));
expect(log).toHaveBeenCalledExactlyOnceWith({ ...context, failureCode }, "Agent lifecycle operation failed");
expect(JSON.stringify(log.mock.calls)).not.toContain("private");
});
});
@@ -0,0 +1,673 @@
import { createAgentLifecycle as createLifecycleCommands } from "../index.js";
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
import { deleteCompany } from "../../../services/company-deletion.js";
import { agentLifecycleCompanyDeletion } from "../company-deletion.js";
import { requireServerAdapter } from "../../../adapters/index.js";
import { agentExecutionsHaveStopped } from "../../../services/agent-execution-stop.js";
import { remoteTerminationReceipt } from "../../../services/remote-execution-termination.js";
import { approvalService } from "../../../services/approvals.js";
import { updateAgentConfigurationInTransaction } from "../../../services/agent-configuration-transaction.js";
import { agentService as agentConfiguration } from "../../../services/agents.js";
import { readFileSync } from "node:fs";
import express from "express";
import request from "supertest";
import { agentRoutes } from "../../../routes/agents.js";
import { errorHandler } from "../../../middleware/error-handler.js";
import { createLifecycleDriver } from "../../../services/agent-lifecycle-driver.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { agents, approvals, projects, issues, heartbeatRuns, nativeRunFinalizations, budgetPolicies, environmentLeases, agentApiKeys, activityLog, agentConfigRevisions, userCompanyPreferences, companies, companyMemberships, principalPermissionGrants, plugins, pluginCompanySettings, createDb, type Db } from "@paperclipai/db";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
import { createAgentLifecycle, configureAgentLifecycle } from "../../../services/agent-lifecycle.js";
import { createLifecycleStore as createStore } from "../adapters/postgres.js";
import { transition } from "../domain/policy.js";
import type { LifecycleAgent } from "../application/ports.js";
import { budgetService } from "../../../services/budgets.js";
const createLifecycleStore = (db: Db) => createStore(db, createAgentLifecycleEffects());
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("agent lifecycle commands", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
let companyId: string;
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }, 90_000);
afterAll(async () => { for (const worker of workers) await worker.stop(); await database?.cleanup(); });
beforeEach(async () => {
for (const work of workers) await work.stop();
companyId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Lifecycle test", issuePrefix: `L${companyId.slice(0, 7)}` });
});
function worker(runHost: (agent: LifecycleAgent) => Promise<"complete" | "pending"> = async () => "complete") {
const value = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost });
workers.push(value); return value;
}
async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); }
async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; }
it("maps existing states without starting setup or clearing pause reasons", async () => {
const rows = await db.insert(agents).values(["idle", "running", "error", "paused", "pending_approval", "terminated"].map(status => ({
companyId, name: status, status, pauseReason: status === "paused" ? "budget" : null,
}))).returning();
const migration = readFileSync(new URL("../../../../../packages/db/src/migrations/0320_daily_onslaught.sql", import.meta.url), "utf8");
await db.execute(sql.raw(migration.slice(migration.indexOf('UPDATE "agents"'))));
for (const row of rows) {
const mapped = await current(row.id);
expect(mapped.lifecycleState).toBe(["paused", "pending_approval", "terminated"].includes(row.status) ? row.status : "ready");
expect(mapped.lifecycleOperation).toBeNull();
expect(mapped.lifecycleRequiredPluginIds).toBeNull();
expect(mapped.lifecycleHolds).toEqual(row.status === "paused" ? ["budget"] : []);
}
});
it("commits the record before preparation and verifies before admission", async () => {
const observed: string[] = [];
const work = worker(async agent => {
expect((await current(agent.id)).lifecycleState).toBe(agent.lifecycleState);
observed.push(agent.lifecycleState); return "complete";
});
const agent = await hire();
expect(agent).toMatchObject({ lifecycleState: "preparing", status: "paused" });
await work.process(agent.id);
expect(observed).toEqual(["preparing", "verifying"]);
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle" });
expect(agent).not.toHaveProperty("lifecycleOperation");
});
it("rolls back a hire and its grants when hire initialization fails", async () => {
const effects = createAgentLifecycleEffects();
const lifecycle = createLifecycleCommands(db, { ...effects,
initializeHire: async (tx, agent, options) => {
await effects.initializeHire(tx, agent, options);
throw new Error("Hire initialization failed");
},
});
await expect(lifecycle.requestHire(companyId, { name: "Rollback hire", adapterType: "process" }))
.rejects.toThrow("Hire initialization failed");
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
expect(await db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId))).toHaveLength(0);
});
it("runs the saved harness test and keeps a failed configuration out of ready", async () => {
const driver = createLifecycleDriver(db, {} as never);
const work = worker(driver.runHost);
const agent = await hire();
await work.process(agent.id);
expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", status: "paused" });
await agentConfiguration(db).update(agent.id, { adapterConfig: { command: process.execPath } });
await work.process(agent.id);
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null });
});
it.each(["claude_local", "gemini_local"])("keeps %s authentication warnings out of ready", async adapterType => {
const adapter = requireServerAdapter(adapterType);
const probe = vi.spyOn(adapter, "testEnvironment").mockResolvedValue({ adapterType, status: "warn",
testedAt: new Date().toISOString(), checks: [{ code: `${adapterType.split("_")[0]}_hello_probe_auth_required`, level: "warn", message: "Login required" }] });
try {
const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Login test", adapterType });
const work = worker(createLifecycleDriver(db, {} as never).runHost);
await work.process(agent.id);
expect(probe).toHaveBeenCalled();
expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." });
} finally { probe.mockRestore(); }
});
it("recovers an expired native owner only after the cancelled run and controller stop", async () => {
const agent = await hire();
const [issue] = await db.insert(issues).values({ companyId, title: "Cancelled native run" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, nativeIssueId: issue.id,
status: "cancelled", invocationSource: "on_demand", runtimeMode: "native",
startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning();
await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id,
phase: "executing", leaseOwner: "stopped-server", leaseExpiresAt: new Date(0), controllerPid: process.pid });
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
await db.update(nativeRunFinalizations).set({ controllerPid: null }).where(eq(nativeRunFinalizations.runId, run.id));
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
await db.update(heartbeatRuns).set({ processPid: 2_000_000_000 }).where(eq(heartbeatRuns.id, run.id));
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, run.id));
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id));
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true);
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]).toMatchObject({ leaseOwner: null, leaseExpiresAt: null });
});
it("does not advance termination while a cancelled process still runs", async () => {
const agent = await hire();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id,
status: "cancelled", invocationSource: "on_demand", runtimeMode: "legacy",
startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning();
const driver = createLifecycleDriver(db, {} as never);
const work = worker(driver.runHost);
await createAgentLifecycle(db).terminateAgent(agent.id);
await work.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("terminating");
await db.update(heartbeatRuns).set({ processPid: null,
resultJson: { executionCancellation: { state: "acknowledged" } } }).where(eq(heartbeatRuns.id, run.id));
await createAgentLifecycle(db).retry(agent.id);
await work.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("terminated");
});
it("does not cancel a reporting agent when its manager pauses", async () => {
const parent = await hire();
const [child] = await db.insert(agents).values({ companyId, name: "Report", reportsTo: parent.id,
status: "idle", lifecycleState: "ready" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: child.id,
status: "queued", invocationSource: "on_demand" }).returning();
const work = worker(createLifecycleDriver(db, {} as never).runHost);
await createAgentLifecycle(db).pauseAgent(parent.id);
await work.process(parent.id);
expect((await current(parent.id)).lifecycleState).toBe("paused");
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]!.status).toBe("queued");
});
it("retains company data until every agent finishes termination", async () => {
const agent = await hire();
const [project] = await db.insert(projects).values({ companyId, name: "Keep until cleanup", leadAgentId: agent.id }).returning();
await db.insert(issues).values({ companyId, projectId: project.id, title: "Keep this task", assigneeAgentId: agent.id });
const pending = worker(async () => "pending");
await expect(deleteCompany(db, companyId)).rejects.toThrow("Complete termination");
expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1);
expect(await db.select().from(issues).where(eq(issues.companyId, companyId))).toHaveLength(1);
await pending.stop();
const complete = worker();
await createAgentLifecycle(db).retry(agent.id);
await complete.process(agent.id);
await deleteCompany(db, companyId);
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(0);
});
it("requires a transaction for the company deletion integration", async () => {
await expect(agentLifecycleCompanyDeletion.deleteCompanyData(db as never, companyId))
.rejects.toThrow("Company deletion requires a database transaction");
});
it("rejects individual deletion before cleanup and rolls back failed cleanup", async () => {
const agent = await hire();
const effects = createAgentLifecycleEffects();
const cleanup = vi.fn(effects.deleteDependencies);
const lifecycle = createLifecycleCommands(db, { ...effects, deleteDependencies: cleanup });
await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Complete termination");
expect(cleanup).not.toHaveBeenCalled();
await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, agent.id));
const [key] = await db.insert(agentApiKeys).values({ companyId, agentId: agent.id, name: "Keep on rollback", keyHash: randomUUID() }).returning();
cleanup.mockImplementation(async (tx, companyId, id) => {
await effects.deleteDependencies(tx, companyId, id);
throw new Error("Dependency cleanup failed");
});
await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Dependency cleanup failed");
expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" });
expect(await db.select().from(agentApiKeys).where(eq(agentApiKeys.id, key.id))).toHaveLength(1);
});
it("deletes terminated and rejected agents only in the requested company", async () => {
await db.insert(agents).values((["terminated", "rejected"] as const).map(lifecycleState => ({
companyId, name: lifecycleState, status: "terminated", lifecycleState,
})));
const [otherCompany] = await db.insert(companies).values({ name: "Keep this company", issuePrefix: `K${companyId.slice(0, 7)}` }).returning();
const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Keep this agent",
status: "terminated", lifecycleState: "terminated" }).returning();
await deleteCompany(db, companyId);
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
expect(await current(otherAgent.id)).toMatchObject({ companyId: otherCompany.id });
expect(await db.select().from(companies).where(eq(companies.id, otherCompany.id))).toHaveLength(1);
});
it("rolls back agent and company data deletion when a later operation fails", async () => {
const agent = await hire("terminated");
const [project] = await db.insert(projects).values({ companyId, name: "Keep on rollback" }).returning();
const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData;
const failure = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => {
await removeAgents(tx, id);
expect(await tx.select().from(agents).where(eq(agents.companyId, id))).toHaveLength(0);
throw new Error("Later deletion failed");
});
try {
await expect(deleteCompany(db, companyId)).rejects.toThrow("Later deletion failed");
expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" });
expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1);
expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(1);
} finally { failure.mockRestore(); }
});
it("blocks a concurrent hire until company deletion commits", async () => {
let reportLock!: (pid: number) => void;
let rejectLock!: (error: unknown) => void;
const locked = new Promise<number>((resolve, reject) => { reportLock = resolve; rejectLock = reject; });
let release!: () => void;
const released = new Promise<void>(resolve => { release = resolve; });
const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData;
const pause = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => {
const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`);
reportLock(backend!.pid);
await released;
await removeAgents(tx, id);
});
const deletion = deleteCompany(db, companyId);
void deletion.catch(rejectLock);
let hiring: Promise<unknown> | undefined;
try {
const pid = await locked;
hiring = hire().then(() => "created", error => error.message);
await vi.waitFor(async () => {
const blocked = await db.execute(sql`select 1 from pg_stat_activity
where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))`);
expect(blocked).toHaveLength(1);
});
release();
await deletion;
expect(await hiring).toBe("Company not found");
expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
} finally {
release();
await Promise.allSettled([deletion, hiring]);
pause.mockRestore();
}
});
it("requires cleanup proof after a remote run is already cancelled", async () => {
const agent = await hire();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id,
status: "cancelled", invocationSource: "on_demand", runtimeMode: "native",
startedAt: new Date(), finishedAt: new Date() }).returning();
const [lease] = await db.insert(environmentLeases).values({ companyId, heartbeatRunId: run.id,
provider: "test", providerLeaseId: "test-lease", status: "pending_cleanup",
cleanupStatus: "failed", releasedAt: new Date() }).returning();
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" })
.where(eq(environmentLeases.id, lease.id));
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false);
await db.update(environmentLeases).set({ metadata: {
remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }),
} }).where(eq(environmentLeases.id, lease.id));
expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true);
});
it("does not prepare a proposed hire until approval", async () => {
const observed: string[] = [];
const work = worker(async agent => { observed.push(agent.lifecycleState); return "complete"; });
const agent = await hire("pending_approval");
await work.process(agent.id);
expect(observed).toEqual([]);
const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" });
await approvalService(db).approve(approval.id, "board");
await vi.waitFor(async () => expect((await current(agent.id)).lifecycleState).toBe("ready"));
expect(observed).toEqual(["preparing", "verifying"]);
});
it.each(["approved", "rejected"] as const)("uses the same %s hire decision for an agent and its approval", async decision => {
const agent = await hire("pending_approval");
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
requestedByUserId: "requester", payload: { agentId: agent.id }, status: "pending" });
const lifecycle = createAgentLifecycle(db);
const decide = decision === "approved" ? lifecycle.approveHire : lifecycle.rejectHire;
const first = await decide(agent.id, "board");
expect(first).toMatchObject({ applied: true, approval: { id: approval.id, status: decision },
agent: { lifecycleState: decision === "approved" ? "preparing" : "rejected" } });
const version = (await current(agent.id)).lifecycleVersion;
expect(await decide({ approvalId: approval.id }, "board")).toMatchObject({ applied: false, hireApprovedAgentId: null });
expect((await current(agent.id)).lifecycleVersion).toBe(version);
});
it("rolls back the approval and agent together when activation fails", async () => {
const agent = await hire("pending_approval");
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
payload: { agentId: agent.id }, status: "pending" });
const lifecycle = createLifecycleCommands(db, { ...createAgentLifecycleEffects(),
recordCreation: async () => { throw new Error("Creation event failed"); } });
await expect(lifecycle.approveHire(agent.id, "board")).rejects.toThrow("Creation event failed");
expect((await current(agent.id)).lifecycleState).toBe("pending_approval");
expect((await db.select().from(approvals).where(eq(approvals.id, approval.id)))[0].status).toBe("pending");
});
it("keeps the hire credential owner and recovers it for migrated approvals", async () => {
for (const savedOwner of [null, "original-owner"]) {
const [agent] = await db.insert(agents).values({ companyId, name: "Migrated hire", status: "pending_approval",
lifecycleState: "pending_approval", lifecycleOperation: savedOwner ? {
id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, responsibleUserId: savedOwner,
} : null }).returning();
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
requestedByUserId: "requesting-member", payload: { agentId: agent.id }, status: "pending" });
await approvalService(db).approve(approval.id, "approving-member");
expect((await current(agent.id)).lifecycleOperation?.responsibleUserId).toBe(savedOwner ?? "requesting-member");
}
});
it("rejects transaction injection and generic state writes", async () => {
const agent = await hire("pending_approval");
await expect(db.transaction(async tx => createAgentLifecycle(tx as unknown as Db))).rejects.toThrow("root database");
await expect(agentConfiguration(db).update(agent.id, { status: "idle" } as never)).rejects.toThrow("lifecycle command");
await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Bypass", lifecycleRequiredPluginIds: [] } as never)).rejects.toThrow("Lifecycle fields");
});
it("clears a rejected hire from the user's primary selection", async () => {
const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Proposed", status: "pending_approval" }, { createdByUserId: "board" });
const preference = () => db.select().from(userCompanyPreferences).where(eq(userCompanyPreferences.companyId, companyId));
expect((await preference())[0]!.primaryAgentId).toBe(agent.id);
const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" });
await approvalService(db).reject(approval.id, "board");
expect((await preference())[0]!.primaryAgentId).toBeNull();
expect((await current(agent.id)).lifecycleState).toBe("rejected");
});
it("rolls back configuration and its revision when the requested transition fails", async () => {
const lifecycle = createAgentLifecycle(db);
const agent = await hire("terminated");
await expect(lifecycle.updateAndTransition(agent.id, "resume", { name: "Must not persist" },
{ recordRevision: { createdByUserId: "board", source: "patch" } })).rejects.toThrow("Cannot resume");
expect((await current(agent.id)).name).toBe("Agent");
expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toEqual([]);
const active = await hire();
expect(await lifecycle.updateAndTransition(active.id, "pause", { name: "Paused agent" })).toMatchObject({ name: "Paused agent", lifecycleState: "pausing" });
});
it("preserves a disabled positive budget at approval and applies explicit configuration changes", async () => {
const agent = await hire("pending_approval");
const budgets = budgetService(db);
await budgets.upsertPolicy(companyId, { scopeType: "agent", scopeId: agent.id, amount: 1000, isActive: false }, null);
const approval = await approvalService(db).create(companyId, { type: "hire_agent",
payload: { agentId: agent.id, budgetMonthlyCents: 1000 }, status: "pending" });
const policy = async () => (await db.select().from(budgetPolicies).where(eq(budgetPolicies.scopeId, agent.id)))[0]!;
await approvalService(db).approve(approval.id, "board");
expect(await policy()).toMatchObject({ amount: 1000, isActive: false });
await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 2000 });
expect(await policy()).toMatchObject({ amount: 2000, isActive: true });
await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 0 });
expect(await policy()).toMatchObject({ amount: 0, isActive: false });
});
it.each(["updateConfiguration", "updateAndTransition"] as const)("limits %s budget reconciliation to the edited agent", async command => {
const lifecycle = createAgentLifecycle(db);
const one = await hire();
const two = await lifecycle.requestHire(companyId, { name: "Other budget target" });
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId));
if (command === "updateConfiguration") await lifecycle.updateConfiguration(one.id, { budgetMonthlyCents: 1000 });
else await lifecycle.updateAndTransition(one.id, "pause", { budgetMonthlyCents: 1000 });
expect((await current(one.id)).lifecycleHolds).toContain("company_paused");
expect((await current(two.id)).lifecycleHolds).toEqual([]);
});
it("limits invocation policy checks to the requested agent", async () => {
const one = await hire();
const two = await createAgentLifecycle(db).requestHire(companyId, { name: "Another agent" });
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId));
await budgetService(db).getInvocationBlock(companyId, one.id);
expect((await current(one.id)).lifecycleHolds).toContain("company_paused");
expect((await current(two.id)).lifecycleHolds).toEqual([]);
});
it("keeps a manual hold when company policy changes", async () => {
const work = worker(); const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id);
await lifecycle.pauseAgent(agent.id); await work.process(agent.id);
await db.update(companies).set({ status: "archived" }).where(eq(companies.id, companyId));
await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id);
await db.update(companies).set({ status: "active" }).where(eq(companies.id, companyId));
await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id);
expect(await current(agent.id)).toMatchObject({ lifecycleState: "paused", lifecycleHolds: ["manual"] });
await lifecycle.resumeAgent(agent.id); await work.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("ready");
});
it("defers automatic policy changes while a task holds the agent lock", async () => {
const agent = await hire("paused");
await db.transaction(async tx => {
await tx.select().from(agents).where(eq(agents.id, agent.id)).for("update");
expect(await createLifecycleStore(db).change(agent.id, "reconcile")).toBeNull();
});
expect(await createLifecycleStore(db).change(agent.id, "reconcile")).not.toBeNull();
});
it("retains the failed phase and the same operation for retry", async () => {
let fail = true;
const work = worker(async agent => { if (agent.lifecycleState === "verifying" && fail) throw new Error("private detail"); return "complete"; });
const agent = await hire(); await work.process(agent.id);
const failed = await current(agent.id);
expect(failed.lifecycleState).toBe("verifying");
expect(failed.lifecycleError).not.toContain("private detail");
fail = false;
await createAgentLifecycle(db).retry(agent.id); await work.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("ready");
});
it("fences a late completion when termination replaces preparation", async () => {
const agent = await hire("paused");
const store = createLifecycleStore(db);
await createAgentLifecycle(db).resumeAgent(agent.id);
const claimed = await store.claim(agent.id, "old", new Date());
expect(claimed).not.toBeNull();
await store.setRequiredPlugins(claimed!, "old", []);
await createAgentLifecycle(db).terminateAgent(agent.id);
expect(await store.completeHost(claimed!, "old")).toBe(false);
expect(await store.change(agent.id, "complete", { owner: "old", version: claimed!.lifecycleVersion })).toBeNull();
expect((await current(agent.id)).lifecycleState).toBe("terminating");
});
it("revokes keys at termination and waits for cleanup before deletion", async () => {
let pending = true;
const phases: string[] = [];
const work = worker(async agent => { phases.push(agent.lifecycleState); return agent.lifecycleState === "cleaning_up" && pending ? "pending" : "complete"; });
const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id);
await db.insert(agentApiKeys).values({ agentId: agent.id, companyId, name: "Test", keyHash: randomUUID() });
await lifecycle.terminateAgent(agent.id); await work.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("cleaning_up");
expect((await db.select().from(agentApiKeys).where(eq(agentApiKeys.agentId, agent.id)))[0].revokedAt).not.toBeNull();
await expect(agentConfiguration(db).remove(agent.id)).rejects.toThrow("Complete termination");
pending = false; await lifecycle.retry(agent.id); await work.process(agent.id);
expect(phases).toContain("terminating");
expect((await current(agent.id)).lifecycleState).toBe("terminated");
await agentConfiguration(db).remove(agent.id);
expect(await current(agent.id)).toBeUndefined();
});
it("retains required plugin IDs and checks the operation and company on each reply", async () => {
const agent = await hire("paused");
const pluginId = randomUUID();
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "lifecycle-test", version: "1.0.0", status: "ready",
manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage"] } as never });
const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete" }));
const driver = createLifecycleDriver(db, { call } as never);
const snapshot = (await createLifecycleStore(db).get(agent.id))!;
expect(await driver.requiredPluginIds(snapshot)).toContain(pluginId);
await db.insert(pluginCompanySettings).values({ companyId, pluginId, enabled: false });
expect(await driver.requiredPluginIds(snapshot)).not.toContain(pluginId);
expect(await driver.requiredPluginIds({ ...snapshot, lifecycleRequiredPluginIds: [pluginId] })).toContain(pluginId);
await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("unavailable");
await db.update(pluginCompanySettings).set({ enabled: true }).where(eq(pluginCompanySettings.pluginId, pluginId));
expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete");
expect(call).toHaveBeenCalledWith(pluginId, "agentLifecycle", { companyId, agentId: agent.id,
operationId: snapshot.lifecycleOperation!.id, version: snapshot.lifecycleVersion, phase: "paused" }, 30_000);
call.mockResolvedValueOnce({ operationId: "stale", version: snapshot.lifecycleVersion, status: "complete" });
await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("Invalid lifecycle result");
});
it("keeps host completion and required plugin results across a worker restart", async () => {
const pluginIds = [randomUUID(), randomUUID()];
const runHost = vi.fn(async () => "complete" as const);
const firstPlugin = vi.fn(async (_agent: LifecycleAgent, pluginId: string) =>
pluginId === pluginIds[0] ? "complete" as const : "pending" as const);
const first = configureAgentLifecycle(db, { requiredPluginIds: async () => pluginIds, runHost, runPlugin: firstPlugin });
workers.push(first);
const agent = await hire();
await first.process(agent.id);
expect(await current(agent.id)).toMatchObject({ lifecycleState: "preparing", lifecycleRequiredPluginIds: pluginIds,
lifecycleOperation: { hostComplete: true, completedPluginIds: [pluginIds[0]] } });
await first.stop();
const discover = vi.fn(async () => []);
const resumedPlugin = vi.fn(async (_agent: LifecycleAgent, _pluginId: string) => "complete" as const);
const resumed = configureAgentLifecycle(db, { requiredPluginIds: discover, runHost, runPlugin: resumedPlugin });
workers.push(resumed);
await createAgentLifecycle(db).retry(agent.id);
await resumed.process(agent.id);
expect((await current(agent.id)).lifecycleState).toBe("ready");
expect(discover).not.toHaveBeenCalled();
expect(runHost).toHaveBeenCalledTimes(2);
expect(resumedPlugin.mock.calls.map(call => call[1])).toEqual([pluginIds[1], ...pluginIds]);
});
it("requires host and plugin completion and freezes the selected plugin IDs", async () => {
const agent = await hire();
const store = createLifecycleStore(db);
const claimed = (await store.claim(agent.id, "test", new Date()))!;
const pluginId = randomUUID();
expect(await store.setRequiredPlugins(claimed, "test", [pluginId])).toBe(true);
expect(await store.setRequiredPlugins(claimed, "test", [])).toBe(false);
expect(await store.completePlugin(claimed, "test", randomUUID())).toBe(false);
const complete = () => store.change(agent.id, "complete", { owner: "test", version: claimed.lifecycleVersion });
expect(await complete()).toBeNull();
expect(await store.completePlugin(claimed, "test", pluginId)).toBe(true);
expect(await complete()).toBeNull();
expect(await store.completeHost(claimed, "test")).toBe(true);
expect(await complete()).toMatchObject({ lifecycleState: "verifying" });
expect(await store.completePlugin(claimed, "test", pluginId)).toBe(false);
});
it("fences verification when the saved configuration changes", async () => {
const lifecycle = createAgentLifecycle(db); const store = createLifecycleStore(db);
const agent = await hire();
const claimed = (await store.claim(agent.id, "configuration-test", new Date()))!;
await store.setRequiredPlugins(claimed, "configuration-test", []);
await agentConfiguration(db).update(agent.id, { adapterConfig: { command: "echo" } });
expect(await store.completeHost(claimed, "configuration-test")).toBe(false);
expect((await lifecycle.get(agent.id))!.lifecycleVersion).toBeGreaterThan(claimed.lifecycleVersion);
});
it("invalidates only changed execution configuration during setup", async () => {
const lifecycle = createAgentLifecycle(db);
const agent = await hire();
const before = await current(agent.id);
await lifecycle.updateConfiguration(agent.id, { name: "Profile only" });
await lifecycle.updateConfiguration(agent.id, { adapterConfig: before.adapterConfig });
expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion);
await lifecycle.updateConfiguration(agent.id, { runtimeConfig: { heartbeat: { enabled: false } } });
expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion + 1);
const work = worker();
await work.process(agent.id);
const ready = await current(agent.id);
expect(ready.lifecycleState).toBe("ready");
await lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } });
expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", lifecycleVersion: ready.lifecycleVersion });
});
it("rejects identity, accounting, and unknown fields through every configuration entry point", async () => {
const agent = await hire();
const lifecycle = createAgentLifecycle(db);
const before = await current(agent.id);
for (const patch of [{ id: randomUUID() }, { companyId: randomUUID() }, { spentMonthlyCents: 0 },
{ spendMonthUtc: "2099-01" }, { lastHeartbeatAt: new Date() }, { errorReason: null },
{ createdAt: new Date() }, { updatedAt: new Date() }, { lifecycleState: "ready" }, { futureColumn: true }]) {
await expect(lifecycle.updateConfiguration(agent.id, patch as never)).rejects.toThrow("Unsupported configuration field");
await expect(lifecycle.updateAndTransition(agent.id, "pause", patch as never)).rejects.toThrow("Unsupported configuration field");
await expect(db.transaction(tx => updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, patch as never)))
.rejects.toThrow("Unsupported configuration field");
}
expect(await current(agent.id)).toEqual(before);
const effects = createAgentLifecycleEffects();
const malformed = createLifecycleCommands(db, { ...effects, prepareConfiguration: async () => ({ id: randomUUID() }) as never });
await expect(malformed.updateConfiguration(agent.id, { name: "Changed" })).rejects.toThrow("Unsupported configuration field");
expect(await current(agent.id)).toEqual(before);
});
it("records only the changed configuration fields for a legacy appearance", async () => {
const [agent] = await db.insert(agents).values({ companyId, name: "Legacy", appearance: null }).returning();
await createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Renamed" },
{ recordRevision: { source: "test", createdByUserId: "board" } });
const revisions = await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id));
expect(revisions).toHaveLength(1);
expect(revisions[0].changedKeys).toEqual(["name"]);
expect(revisions[0].beforeConfig.appearance).toEqual(revisions[0].afterConfig.appearance);
});
it("accepts the unchanged resolved appearance of a pending hire", async () => {
const [agent] = await db.insert(agents).values({ companyId, name: "Pending legacy", status: "pending_approval", appearance: null }).returning();
const resolved = await agentConfiguration(db).getById(agent.id);
await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { appearance: resolved!.appearance }))
.resolves.toMatchObject({ id: agent.id });
await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Changed" }))
.rejects.toThrow("Pending approval agent configuration cannot be changed");
});
it("rolls back a root configuration command when dependent records fail", async () => {
const agent = await hire();
const before = await current(agent.id);
const effects = createAgentLifecycleEffects();
const lifecycle = createLifecycleCommands(db, { ...effects, completeConfiguration: async (...args) => {
await effects.completeConfiguration(...args);
throw new Error("Configuration completion failed");
} });
await expect(lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } },
{ recordRevision: { source: "test", createdByUserId: "board" } })).rejects.toThrow("Configuration completion failed");
expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig,
lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation });
expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toHaveLength(0);
});
it("rolls back configuration and verification invalidation in the binding transaction", async () => {
const agent = await hire();
const before = await current(agent.id);
await expect(updateAgentConfigurationInTransaction(db, agent.id, { name: "Uncommitted" }))
.rejects.toThrow("requires a database transaction");
expect(() => updateAgentConfigurationInTransaction(db, agent.id, { budgetMonthlyCents: 100 } as never))
.toThrow("Budget changes require a root lifecycle command");
await expect(db.transaction(async tx => agentConfiguration(tx as unknown as Db).update(agent.id, { name: "Uncommitted" })))
.rejects.toThrow("root database");
await expect(db.transaction(async tx => {
await updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, { adapterConfig: { command: "echo" } });
expect((await tx.select().from(agents).where(eq(agents.id, agent.id)))[0].lifecycleVersion).toBe(before.lifecycleVersion + 1);
throw new Error("Binding change failed");
})).rejects.toThrow("Binding change failed");
expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig,
lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation });
});
it("permits a board retry and rejects an agent or a different company", async () => {
const agent = await hire();
function app(actor: Express.Request["actor"]) {
const app = express(); app.use(express.json());
app.use((req, _res, next) => { req.actor = actor; next(); });
app.use("/api", agentRoutes(db)); app.use(errorHandler); return app;
}
const board: Express.Request["actor"] = { type: "board", source: "session", userId: "board", companyIds: [companyId],
memberships: [{ companyId, membershipRole: "admin", status: "active" }] };
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "admin" });
await db.insert(principalPermissionGrants).values(["agents:create", "agents:configure"].map(permissionKey => ({ companyId, principalType: "user", principalId: "board", permissionKey })));
const url = `/api/agents/${agent.id}/lifecycle/retry`;
const response = await request(app(board)).post(url).send({});
expect(response.status, JSON.stringify(response.body)).toBe(200);
expect(response.body.lifecycleState).toBe("preparing");
expect(response.body).not.toHaveProperty("lifecycleOperation");
expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, agent.id), eq(activityLog.action, "agent.lifecycle_retried"))))
.toEqual([expect.objectContaining({ actorType: "user", actorId: "board" })]);
await request(app({ type: "agent", source: "agent_key", companyId, agentId: agent.id })).post(url).send({}).expect(403);
const denied = await request(app({ ...board, companyIds: [], memberships: [] })).post(url).send({});
expect([403, 404]).toContain(denied.status);
});
it("recovers an expired lease and never accepts an expired completion", async () => {
const agent = await hire("paused"); const store = createLifecycleStore(db);
await createAgentLifecycle(db).resumeAgent(agent.id);
const claim = await store.claim(agent.id, "first", new Date(Date.now() - 180_000));
expect(claim).not.toBeNull();
await store.setRequiredPlugins(claim!, "first", []);
await store.completeHost(claim!, "first");
expect(await store.change(agent.id, "complete", { owner: "first", version: claim!.lifecycleVersion })).toBeNull();
expect(await store.claim(agent.id, "second", new Date())).not.toBeNull();
});
});
describe("lifecycle transition rules", () => {
it("does not bypass authorization, verification, or cleanup", () => {
expect(() => transition("pending_approval", "resume")).toThrow();
expect(transition("preparing", "complete")).toBe("verifying");
expect(transition("terminating", "complete")).toBe("cleaning_up");
expect(transition("cleaning_up", "complete")).toBe("terminated");
expect(() => transition("terminated", "resume")).toThrow();
expect(transition("terminating", "terminate")).toBe("terminating");
});
});
@@ -0,0 +1,63 @@
import { agents, companies, createDb, type Db } from "@paperclipai/db";
import { asc, eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { startAgentLifecycle } from "../index.js";
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("lifecycle policy sweep", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("paperclip-lifecycle-policy-");
db = createDb(database.connectionString);
}, 90_000);
afterAll(async () => { await database?.cleanup(); });
it("caps each policy page, skips ineligible agents, and returns to the first page", async () => {
const [company] = await db.insert(companies).values({ name: "Policy pages", issuePrefix: "POL" }).returning();
const eligible = await db.insert(agents).values(Array.from({ length: 101 }, (_, index) => ({
id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`,
companyId: company.id, name: `Agent ${index}`, status: "idle", lifecycleState: "ready" as const,
lifecycleHolds: ["company_paused"], pauseReason: "company_paused",
}))).returning();
const excluded = await db.insert(agents).values((["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"] as const).map(lifecycleState => ({
companyId: company.id, name: lifecycleState, status: "terminated", lifecycleState,
lifecycleHolds: ["company_paused"], pauseReason: "company_paused",
}))).returning();
const effects = createAgentLifecycleEffects();
const transaction = vi.spyOn(effects, "transaction");
const now = Date.now();
const clock = vi.spyOn(Date, "now").mockReturnValue(now);
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => ({ unref() {} }) as ReturnType<typeof setInterval>);
const clearTimer = vi.spyOn(globalThis, "clearInterval").mockImplementation(() => {});
const work = startAgentLifecycle(db, effects, {
requiredPluginIds: async () => [], runHost: async () => "pending", runPlugin: async () => "pending",
}, () => true);
const rows = () => db.select().from(agents).where(eq(agents.companyId, company.id)).orderBy(asc(agents.id));
try {
await work.sweep();
expect(transaction).toHaveBeenCalledTimes(100);
expect((await rows()).filter(row => row.lifecycleHolds.length === 0)).toHaveLength(100);
await work.sweep();
expect(transaction).toHaveBeenCalledTimes(100);
clock.mockReturnValue(now + 60_001);
await work.sweep();
expect(transaction).toHaveBeenCalledTimes(101);
expect((await rows()).filter(row => row.lifecycleHolds.length > 0).map(row => row.id).sort())
.toEqual(excluded.map(row => row.id).sort());
await db.update(agents).set({ lifecycleHolds: ["company_paused"] }).where(eq(agents.id, eligible[0].id));
clock.mockReturnValue(now + 120_002);
await work.sweep();
expect(transaction).toHaveBeenCalledTimes(201);
expect((await rows())[0].lifecycleHolds).toEqual([]);
} finally {
await work.stop();
transaction.mockRestore();
clock.mockRestore();
timer.mockRestore();
clearTimer.mockRestore();
}
});
});
@@ -0,0 +1,96 @@
import { agentAppearanceSchema } from "@paperclipai/shared";
import { approvals, type Db } from "@paperclipai/db";
import { and, desc, eq, inArray, sql } from "drizzle-orm";
import { unprocessable } from "../../../errors.js";
import { approvalResolutionRecords, resolvableApprovalStatuses } from "../../../lib/approval-records.js";
import { agentRecords } from "./records.js";
import { createLifecycleStore } from "./postgres.js";
import type { LifecycleEffects } from "./effects.js";
export type HireDecisionTarget = string | { approvalId: string };
type HireDecisionResult = {
approval: typeof approvals.$inferSelect | null;
agent: Awaited<ReturnType<ReturnType<typeof agentRecords>["getById"]>>;
applied: boolean;
activated: boolean;
hireApprovedAgentId: string | null;
};
export function hireApprovalService(db: Db, effects: LifecycleEffects) {
const { getExistingApproval, resolveApproval } = approvalResolutionRecords(db);
async function decide(target: HireDecisionTarget, status: "approved" | "rejected", userId: string, note?: string | null): Promise<HireDecisionResult | null> {
const initial = typeof target === "string"
? await agentRecords(db, effects).getById(target)
: await getExistingApproval(target.approvalId);
if (!initial) return null;
if (typeof target !== "string" && (!("type" in initial) || initial.type !== "hire_agent")) throw unprocessable("Expected a hire approval");
const companyId = initial.companyId;
// Hire decisions acquire the company lock before changing an approval or agent.
return effects.transaction(db, companyId, async (tx, publications) => {
const records = agentRecords(tx, effects);
let approvalId = typeof target === "string" ? null : target.approvalId;
if (typeof target === "string") {
const [open] = await tx.select({ id: approvals.id }).from(approvals).where(and(
eq(approvals.companyId, companyId), eq(approvals.type, "hire_agent"),
inArray(approvals.status, resolvableApprovalStatuses),
sql`${approvals.payload}->>'agentId' = ${target}`,
)).orderBy(desc(approvals.createdAt)).limit(1);
approvalId = open?.id ?? null;
}
let approval: typeof approvals.$inferSelect | null = null;
let applied = true;
if (approvalId) {
const result = await resolveApproval(approvalId, status, userId, note, tx);
approval = result.approval;
applied = result.applied;
}
const payload = approval?.payload ?? {};
let agentId = typeof target === "string" ? target : typeof payload.agentId === "string" ? payload.agentId : null;
if (applied && status === "approved") {
if (agentId) {
const activated = await records.activatePendingApproval(agentId, approval ? payload : null, approval ? approval.requestedByUserId : userId);
if (!approval) applied = activated?.activated ?? false;
} else {
const created = await records.create(companyId, {
name: String(payload.name ?? "New Agent"),
appearance: payload.appearance == null ? undefined : agentAppearanceSchema.parse(payload.appearance),
role: String(payload.role ?? "general"),
title: typeof payload.title === "string" ? payload.title : null,
reportsTo: typeof payload.reportsTo === "string" ? payload.reportsTo : null,
capabilities: typeof payload.capabilities === "string" ? payload.capabilities : null,
adapterType: String(payload.adapterType ?? "process"),
adapterConfig:
typeof payload.adapterConfig === "object" && payload.adapterConfig !== null
? (payload.adapterConfig as Record<string, unknown>)
: {},
budgetMonthlyCents:
typeof payload.budgetMonthlyCents === "number" ? payload.budgetMonthlyCents : 0,
metadata:
typeof payload.metadata === "object" && payload.metadata !== null
? (payload.metadata as Record<string, unknown>)
: null,
status: "idle",
spentMonthlyCents: 0,
permissions: undefined,
lastHeartbeatAt: null,
}, { runnerResolved: true, createdByUserId: approval!.requestedByAgentId ? null : approval!.requestedByUserId, responsibleUserId: approval!.requestedByUserId });
agentId = created.id;
}
if (approval && agentId && typeof payload.budgetMonthlyCents === "number" && payload.budgetMonthlyCents > 0) {
await effects.setAgentBudget(tx, publications, companyId, agentId, payload.budgetMonthlyCents, userId);
}
} else if (applied && agentId) {
if (approval) await records.rejectPendingHire(agentId);
else {
const before = await records.getById(agentId);
await createLifecycleStore(tx, effects).change(agentId, "reject");
applied = before?.lifecycleState !== "rejected";
}
}
return { approval, agent: agentId ? await records.getById(agentId) : null, applied,
activated: status === "approved" && applied,
hireApprovedAgentId: status === "approved" && applied ? agentId : null };
});
}
return { decide };
}
@@ -0,0 +1,19 @@
import { agents } from "@paperclipai/db";
import { and, eq, inArray } from "drizzle-orm";
import type { CompanyDeletionParticipant } from "../../../lib/company-deletion.js";
import { AgentLifecycleConflict, assertAgentPurgeAllowed } from "../domain/policy.js";
export const agentLifecycleCompanyDeletion: CompanyDeletionParticipant = {
async deleteCompanyData(tx, companyId) {
if (!("nestedIndex" in tx)) {
throw new AgentLifecycleConflict("Company deletion requires a database transaction");
}
const rows = await tx.select({ lifecycleState: agents.lifecycleState }).from(agents)
.where(eq(agents.companyId, companyId)).for("update");
for (const agent of rows) assertAgentPurgeAllowed(agent);
await tx.delete(agents).where(and(
eq(agents.companyId, companyId),
inArray(agents.lifecycleState, ["terminated", "rejected"]),
));
},
};
@@ -0,0 +1,32 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { agents, type Db } from "@paperclipai/db";
import { agentRecordQueries, AGENT_CONFIGURATION_FIELDS, jsonEqual, type AgentConfigurationPatch, type UpdateAgentOptions } from "../../../lib/agent-records.js";
import type { ActivityPublication } from "../../../types/activity-publication.js";
import { AgentLifecycleConflict } from "../domain/policy.js";
import type { LifecycleEffects } from "./effects.js";
export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects, id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions, publications: ActivityPublication[] = []) {
if (!("nestedIndex" in tx)) throw new AgentLifecycleConflict("Configuration integration requires a database transaction");
assertConfigurationFields(data);
const [current] = await tx.select().from(agents).where(eq(agents.id, id)).for("update");
if (!current) return null;
const patch = await effects.prepareConfiguration(tx, current, data, options);
assertConfigurationFields(patch);
const changed = ["adapterType", "adapterConfig", "runtimeConfig", "defaultEnvironmentId"].some(key =>
Object.prototype.hasOwnProperty.call(patch, key) && !jsonEqual(patch[key as keyof typeof patch], current[key as keyof typeof current]));
const invalidate = changed && ["preparing", "verifying", "resuming"].includes(current.lifecycleState);
const [updated] = await tx.update(agents).set({ ...patch, updatedAt: new Date(), ...(invalidate ? {
lifecycleVersion: current.lifecycleVersion + 1, lifecycleError: null,
lifecycleOperation: { ...current.lifecycleOperation!, id: randomUUID(), hostComplete: false,
completedPluginIds: [], leaseOwner: undefined, leaseUntil: undefined, retryAt: undefined },
} : {}) }).where(eq(agents.id, id)).returning();
await effects.completeConfiguration(tx, current, updated, patch, options, publications);
return agentRecordQueries(tx).getById(id);
}
function assertConfigurationFields(data: AgentConfigurationPatch) {
if (Object.keys(data).some(field => !AGENT_CONFIGURATION_FIELDS.some(allowed => allowed === field))) {
throw new AgentLifecycleConflict("Unsupported configuration field; use the command that owns this field, such as a lifecycle command");
}
}
@@ -0,0 +1,27 @@
import { agents, type Db } from "@paperclipai/db";
import { and, eq, inArray } from "drizzle-orm";
import { conflict } from "../../../errors.js";
import { agentRecordQueries } from "../../../lib/agent-records.js";
import { readBuiltInAgentMarker } from "../../../lib/built-in-agent-metadata.js";
import { assertAgentPurgeAllowed } from "../domain/policy.js";
import type { LifecycleEffects } from "./effects.js";
export async function purgeAgent(db: Db, effects: LifecycleEffects, id: string) {
const { getById, normalizeAgentRow } = agentRecordQueries(db);
const existing = await getById(id);
if (!existing) return null;
return effects.transaction(db, existing.companyId, async tx => {
const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update");
if (!agent) return null;
const marker = readBuiltInAgentMarker(agent.metadata);
if (marker) throw conflict("Built-in agents cannot be deleted; pause them instead", {
code: "built_in_agent_undeletable", key: marker.key, featureKeys: marker.featureKeys,
});
assertAgentPurgeAllowed(agent);
await effects.deleteDependencies(tx, agent.companyId, id);
const [deleted] = await tx.delete(agents).where(and(
eq(agents.id, id), inArray(agents.lifecycleState, ["terminated", "rejected"]),
)).returning();
return deleted ? normalizeAgentRow(deleted) : null;
});
}
@@ -0,0 +1,24 @@
import type { LifecycleFailure } from "../application/ports.js";
import type { Db } from "@paperclipai/db";
import type { budgetPolicies } from "@paperclipai/db";
import type { CreateAgentData, CreateAgentOptions, AgentConfigurationPatch, AgentConfigurationRecord, AgentHireRecord, UpdateAgentOptions } from "../../../lib/agent-records.js";
import type { ActivityPublication } from "../../../types/activity-publication.js";
/** Services supply integrations; lifecycle owns the transaction and state writes. */
export interface LifecycleEffects {
reportFailure(context: LifecycleFailure, error: unknown): void;
deleteDependencies(db: Db, companyId: string, agentId: string): Promise<void>;
prepareConfiguration(db: Db, current: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch>;
completeConfiguration(db: Db, before: AgentConfigurationRecord, after: AgentConfigurationRecord, patch: AgentConfigurationPatch, options: UpdateAgentOptions | undefined, publications: ActivityPublication[]): Promise<void>;
transaction<T>(db: Db, companyId: string, work: (tx: Db, publications: ActivityPublication[]) => Promise<T>): Promise<T>;
policyBlocks(db: Db, policy: typeof budgetPolicies.$inferSelect): Promise<boolean>;
setAgentBudget(db: Db, publications: ActivityPublication[], companyId: string, agentId: string, amount: number, userId: string | null, isActive?: boolean): Promise<unknown>;
enforceBudget(db: Db, companyId: string, agentId: string): Promise<void>;
recordCreation(db: Db, companyId: string, agentId: string): Promise<void>;
recordStatus(db: Db, companyId: string, agentId: string, before: string, after: string): Promise<void>;
clearPrimary(db: Db, companyId: string, agentId: string): Promise<unknown>;
prepareHire(db: Db, companyId: string, data: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData>;
initializeHire(db: Db, agent: AgentHireRecord, options?: CreateAgentOptions): Promise<void>;
prepareHireApproval(db: Db, agent: AgentHireRecord, payload?: Record<string, unknown> | null): Promise<AgentConfigurationPatch>;
completeHireApproval(db: Db, agent: AgentHireRecord, previous: AgentHireRecord): Promise<void>;
}
@@ -0,0 +1,151 @@
import { randomUUID } from "node:crypto";
import { and, asc, eq, inArray, sql } from "drizzle-orm";
import { agents, agentApiKeys, companies, budgetPolicies, type Db } from "@paperclipai/db";
import type { AgentLifecycleOperation } from "@paperclipai/shared";
import { AgentLifecycleConflict, compatibilityStatus, transition } from "../domain/policy.js";
import type { LifecycleEffects } from "./effects.js";
import type { LifecycleStore } from "../application/ports.js";
export function assertRootDatabase(db: Db) {
if ("nestedIndex" in db) throw new AgentLifecycleConflict("Lifecycle commands require the root database connection");
}
export function createLifecycleStore(db: Db, effects: LifecycleEffects): LifecycleStore {
const get = (id: string) => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null);
return {
get,
async change(id, command, options = {}) {
const existing = await get(id);
if (!existing) return null;
return effects.transaction(db, existing.companyId, async tx => {
const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update", command === "reconcile" ? { skipLocked: true } : undefined);
if (!agent) return null;
if (options.version !== undefined && options.version !== agent.lifecycleVersion) return null;
if (command === "complete" && (!options.owner || !agent.lifecycleOperation?.leaseUntil || agent.lifecycleOperation?.leaseOwner !== options.owner ||
Date.parse(agent.lifecycleOperation?.leaseUntil ?? "") <= Date.now() ||
!agent.lifecycleOperation.hostComplete || agent.lifecycleRequiredPluginIds === null ||
agent.lifecycleRequiredPluginIds.some(id => !agent.lifecycleOperation!.completedPluginIds.includes(id)))) return null;
if (command === "retry") {
transition(agent.lifecycleState, command);
if (agent.lifecycleOperation?.leaseUntil && Date.parse(agent.lifecycleOperation.leaseUntil) > Date.now()) return agent;
const [updated] = await tx.update(agents).set({ lifecycleError: null,
lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, id)).returning();
return updated;
}
if (command === "resume" && !["ready", "paused", "pausing", "resuming"].includes(agent.lifecycleState)) transition(agent.lifecycleState, command);
let holds = agent.lifecycleHolds;
const reason = options.reason ?? "manual";
if (command === "reconcile") {
if (["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"].includes(agent.lifecycleState)) return agent;
const [company] = await tx.select().from(companies).where(eq(companies.id, agent.companyId));
const policies = await tx.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, agent.companyId),
eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, id)));
let budgetBlocked = false;
for (const policy of policies) if (await effects.policyBlocks(tx, policy)) budgetBlocked = true;
holds = holds.filter(hold => hold !== "budget" && hold !== "company_archived" && hold !== "company_paused");
if (budgetBlocked) holds.push("budget");
if (company.status === "archived") holds.push("company_archived");
if (company.status === "paused") holds.push("company_paused");
}
if (command === "pause") holds = [...new Set([...holds, reason])];
if (command === "resume") holds = holds.filter(hold => reason === "user" ? ["budget", "company_archived", "company_paused"].includes(hold) : hold !== reason);
let next = command === "reconcile"
? holds.length && !["pausing", "paused"].includes(agent.lifecycleState) ? "pausing" as const
: !holds.length && agent.lifecycleState === "paused" ? "resuming" as const : agent.lifecycleState
: command === "resume" && (holds.length || agent.lifecycleState === "pausing") ? agent.lifecycleState
: transition(agent.lifecycleState, command, agent.lifecycleOperation?.resumeState);
if (command === "complete" && next === "ready" && holds.length) next = "pausing";
if (command === "complete" && next === "paused" && !holds.length) next = "resuming";
if (next === agent.lifecycleState) {
if (JSON.stringify(holds) !== JSON.stringify(agent.lifecycleHolds) || agent.pauseReason !== (holds[0] ?? null)) {
await tx.update(agents).set({ lifecycleHolds: holds, pauseReason: holds[0] ?? null }).where(eq(agents.id, id));
}
return agent;
}
const operation: AgentLifecycleOperation = {
responsibleUserId: agent.lifecycleOperation?.responsibleUserId,
id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0,
resumeState: next === "pausing"
? agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying" ? agent.lifecycleState : agent.lifecycleOperation?.resumeState ?? "ready"
: agent.lifecycleOperation?.resumeState,
};
if (next === "ready") operation.resumeState = "ready";
const status = compatibilityStatus(next, agent.status);
const [updated] = await tx.update(agents).set({
lifecycleHolds: holds, lifecycleState: next, lifecycleVersion: agent.lifecycleVersion + 1, lifecycleError: null,
lifecycleOperation: operation, status,
pauseReason: holds[0] ?? null,
pausedAt: next === "pausing" ? new Date() : next === "ready" ? null : agent.pausedAt,
updatedAt: new Date(),
}).where(eq(agents.id, id)).returning();
if (next === "terminating" || next === "rejected") {
await tx.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id));
await effects.clearPrimary(tx, agent.companyId, id);
}
if (command === "approve") await effects.recordCreation(tx, agent.companyId, id);
if (next === "pausing" || next === "resuming" || next === "terminating" || next === "rejected") {
await effects.recordStatus(tx, agent.companyId, id,
next === "resuming" ? "paused" : "idle",
next === "resuming" ? "idle" : next === "pausing" ? "paused" : "terminated");
}
return updated;
});
},
async claim(id, owner, now) {
const [agent] = await db.update(agents).set({
lifecycleOperation: sql`coalesce(${agents.lifecycleOperation}, '{"id":"", "hostComplete":false, "completedPluginIds":[], "attempts":0}'::jsonb)
|| jsonb_build_object('leaseOwner', ${owner}::text, 'leaseUntil', ${(new Date(now.getTime() + 120_000)).toISOString()}::text,
'attempts', coalesce((${agents.lifecycleOperation}->>'attempts')::integer, 0) + 1)`,
}).where(and(eq(agents.id, id), inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]),
sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
)).returning();
return agent ?? null;
},
async renew(agent, owner, now) {
const rows = await db.update(agents).set({ lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{leaseUntil}', ${JSON.stringify(new Date(now.getTime() + 120_000).toISOString())}::jsonb)` })
.where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
return rows.length > 0;
},
async setRequiredPlugins(agent, owner, pluginIds) {
const rows = await db.update(agents).set({ lifecycleRequiredPluginIds: pluginIds })
.where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
sql`${agents.lifecycleRequiredPluginIds} is null`,
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
return rows.length > 0;
},
async completeHost(agent, owner) {
const rows = await db.update(agents).set({
lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{hostComplete}', 'true'::jsonb)`,
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
return rows.length > 0;
},
async completePlugin(agent, owner, pluginId) {
const rows = await db.update(agents).set({
lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{completedPluginIds}',
coalesce(${agents.lifecycleOperation}->'completedPluginIds', '[]'::jsonb) || ${JSON.stringify([pluginId])}::jsonb)`,
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
sql`${agents.lifecycleRequiredPluginIds} @> ${JSON.stringify([pluginId])}::jsonb`,
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
return rows.length > 0;
},
async defer(agent, owner, error, now) {
const rows = await db.update(agents).set({
lifecycleError: error,
lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil')
|| jsonb_build_object('retryAt', ${new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString()}::text)`,
updatedAt: now,
}).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
return rows.length > 0;
},
async pending(limit, now) {
return db.select({ id: agents.id }).from(agents).where(and(
inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]),
sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`,
)).orderBy(asc(agents.updatedAt), asc(agents.id)).limit(limit).then(rows => rows.map(row => row.id));
},
};
}
@@ -0,0 +1,118 @@
import type { LifecycleEffects } from "./effects.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { agents, agentApiKeys, type Db } from "@paperclipai/db";
import { conflict, notFound } from "../../../errors.js";
import { agentRecordQueries, type CreateAgentOptions, type CreateAgentData } from "../../../lib/agent-records.js";
export function agentRecords(db: Db, effects: LifecycleEffects) {
const { getById } = agentRecordQueries(db);
return {
getById,
create: async (companyId: string, data: CreateAgentData, options?: CreateAgentOptions) => {
if (Object.keys(data).some(key => key.startsWith("lifecycle"))) throw conflict("Lifecycle fields belong to the lifecycle module");
return effects.transaction(db, companyId, async (txDb) => {
const tx = txDb;
const prepared = await effects.prepareHire(txDb, companyId, data, options);
const created = await tx
.insert(agents)
.values({
...prepared,
lifecycleState: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : data.status === "paused" ? "paused" : "preparing",
lifecycleVersion: 1,
lifecycleHolds: data.status === "paused" ? [data.pauseReason ?? "manual"] : [],
pauseReason: data.status === "paused" ? data.pauseReason ?? "manual" : null,
pausedAt: data.status === "paused" ? new Date() : null,
lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, resumeState: "preparing", responsibleUserId: options?.responsibleUserId ?? options?.createdByUserId },
status: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : "paused",
companyId,
})
.returning()
.then((rows) => rows[0]);
await effects.initializeHire(txDb, created, options);
if (created.status !== "pending_approval" && created.status !== "terminated") {
await effects.recordCreation(txDb, companyId, created.id);
}
const normalizedCreated = await agentRecordQueries(txDb).getById(created.id);
if (!normalizedCreated) {
throw notFound("Agent not found");
}
return normalizedCreated;
});
},
clearError: async (id: string) => {
const existing = await getById(id);
if (!existing) return null;
if (existing.status === "terminated") throw conflict("Cannot clear error on terminated agent");
if (existing.status === "pending_approval") {
throw conflict("Pending approval agents cannot have errors cleared");
}
if (existing.status !== "error") {
throw conflict("Only agents in error status can have their error cleared");
}
const updated = await db
.update(agents)
.set({
status: "idle",
pauseReason: null,
pausedAt: null,
errorReason: null,
updatedAt: new Date(),
})
.where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
.returning()
.then((rows) => rows[0] ?? null);
if (!updated) {
throw conflict("Only agents in error status can have their error cleared");
}
return getById(updated.id);
},
rejectPendingHire: async (id: string) => {
const rows = await db.update(agents).set({ status: "terminated", lifecycleState: "rejected",
lifecycleVersion: sql`${agents.lifecycleVersion} + 1`, lifecycleOperation: null,
updatedAt: new Date() }).where(and(eq(agents.id, id), eq(agents.lifecycleState, "pending_approval"))).returning();
for (const row of rows) {
await effects.clearPrimary(db, row.companyId, id);
await db.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id));
await effects.recordStatus(db, row.companyId, id, "pending_approval", "terminated");
}
return rows;
},
activatePendingApproval: async (id: string, approvedPayload?: Record<string, unknown> | null, requestedByUserId?: string | null) => {
const activatedAgent = await db.transaction(async (tx) => {
const txDb = tx as unknown as Db;
const existing = await agentRecordQueries(txDb).getById(id);
if (!existing || existing.status !== "pending_approval") return null;
const patch = await effects.prepareHireApproval(txDb, existing, approvedPayload);
const updated = await tx
.update(agents)
.set({ ...patch, status: "paused", lifecycleState: "preparing", lifecycleVersion: existing.lifecycleVersion + 1, lifecycleError: null, lifecycleOperation: sql`jsonb_build_object('id', ${randomUUID()}::text, 'hostComplete', false, 'completedPluginIds', '[]'::jsonb, 'attempts', 0, 'responsibleUserId', coalesce(nullif(${agents.lifecycleOperation}->'responsibleUserId', 'null'::jsonb), to_jsonb(${requestedByUserId ?? null}::text)))`, updatedAt: new Date() })
.where(and(eq(agents.id, id), eq(agents.status, "pending_approval")))
.returning()
.then((rows) => rows[0] ?? null);
if (!updated) return null;
await effects.completeHireApproval(txDb, updated, existing);
await effects.recordCreation(txDb, existing.companyId, updated.id);
const agent = await agentRecordQueries(txDb).getById(updated.id);
if (!agent) {
throw notFound("Agent not found");
}
return agent;
});
if (activatedAgent) {
return { agent: activatedAgent, activated: true };
}
const existing = await getById(id);
return existing ? { agent: existing, activated: false } : null;
},
};
}
@@ -0,0 +1,41 @@
import type { AgentLifecycleOperation, AgentLifecycleState } from "@paperclipai/shared";
import type { LifecycleCommand } from "../domain/policy.js";
export interface LifecycleAgent {
id: string;
companyId: string;
lifecycleState: AgentLifecycleState;
lifecycleVersion: number;
lifecycleRequiredPluginIds: string[] | null;
lifecycleError: string | null;
lifecycleOperation: AgentLifecycleOperation | null;
}
export interface LifecycleStore {
get(id: string): Promise<LifecycleAgent | null>;
change(id: string, command: LifecycleCommand, options?: { version?: number; owner?: string; reason?: string }): Promise<LifecycleAgent | null>;
claim(id: string, owner: string, now: Date): Promise<LifecycleAgent | null>;
renew(agent: LifecycleAgent, owner: string, now: Date): Promise<boolean>;
setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise<boolean>;
completeHost(agent: LifecycleAgent, owner: string): Promise<boolean>;
completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise<boolean>;
defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date): Promise<boolean>;
pending(limit: number, now: Date): Promise<string[]>;
}
export interface LifecycleDriver {
requiredPluginIds(agent: LifecycleAgent): Promise<string[]>;
runHost(agent: LifecycleAgent): Promise<"complete" | "pending">;
runPlugin(agent: LifecycleAgent, pluginId: string): Promise<"complete" | "pending">;
}
export interface LifecycleFailure {
stage: "claim" | "renew" | "select_plugins" | "save_plugins" | "host" | "plugin"
| "complete_host" | "complete_plugin" | "transition" | "defer" | "scan" | "policy_scan";
agentId?: string;
companyId?: string;
phase?: AgentLifecycleState;
operationId?: string;
version?: number;
pluginId?: string;
}
@@ -0,0 +1,96 @@
import { randomUUID } from "node:crypto";
import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js";
import { isLifecycleWorkPending } from "../domain/policy.js";
/** Database leases protect each attempt across server processes. */
export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) {
let stopped = false;
const active = new Map<string, Promise<void>>();
let running: Promise<void> | undefined;
async function attempt<T>(context: LifecycleFailure, work: () => Promise<T>): Promise<T> {
try { return await work(); }
catch (error) { reportFailure(context, error); throw error; }
}
async function process(id: string) {
phase: for (let step = 0; !stopped && canRun() && step < 8; step++) {
const owner = randomUUID();
const agent = await attempt({ stage: "claim", agentId: id }, () => store.claim(id, owner, new Date()));
if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return;
const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState,
operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion };
const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) =>
attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work);
const renewal = setInterval(() => {
void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {});
}, 30_000);
renewal.unref?.();
try {
const operation = agent.lifecycleOperation!;
const pluginIds = agent.lifecycleRequiredPluginIds ?? await runStep("select_plugins", () => driver.requiredPluginIds(agent));
if (agent.lifecycleRequiredPluginIds === null && !await runStep("save_plugins", () => store.setRequiredPlugins(agent, owner, pluginIds))) continue;
if (!canRun()) {
await runStep("defer", () => store.defer(agent, owner, null, new Date()));
return;
}
if (!operation.hostComplete) {
if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue;
if (await runStep("host", () => driver.runHost(agent)) === "pending") {
if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue;
return;
}
if (!await runStep("complete_host", () => store.completeHost(agent, owner))) continue;
}
for (const pluginId of pluginIds) {
if (operation.completedPluginIds.includes(pluginId)) continue;
if (!canRun()) {
await runStep("defer", () => store.defer(agent, owner, null, new Date()));
return;
}
if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue phase;
if (await runStep("plugin", () => driver.runPlugin(agent, pluginId), pluginId) === "pending") {
if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue phase;
return;
}
if (!await runStep("complete_plugin", () => store.completePlugin(agent, owner, pluginId), pluginId)) continue phase;
}
const changed = await runStep("transition", () => store.change(id, "complete", { version: agent.lifecycleVersion, owner }));
if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return;
} catch {
if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue;
return;
} finally {
clearInterval(renewal);
}
}
}
async function sweep() {
const pending = await attempt({ stage: "scan" }, () => store.pending(100, new Date()));
const entries = pending.values();
await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => {
for (const id of entries) await start(id);
}));
}
function start(id: string): Promise<void> {
if (stopped || !canRun()) return Promise.resolve();
const existing = active.get(id);
if (existing) return existing;
const work = process(id).finally(() => { active.delete(id); });
active.set(id, work);
return work;
}
return {
process: start,
async stop() {
stopped = true;
await Promise.allSettled([...active.values()]);
},
sweep() {
return running ??= sweep().finally(() => { running = undefined; });
},
};
}
@@ -0,0 +1 @@
export { agentLifecycleCompanyDeletion } from "./adapters/company-deletion.js";
@@ -0,0 +1 @@
export { updateAgentConfiguration } from "./adapters/configuration.js";
@@ -0,0 +1,63 @@
import type { AgentLifecycleState } from "@paperclipai/shared";
export type LifecycleCommand = "reconcile" | "approve" | "reject" | "pause" | "resume" | "terminate" | "complete" | "retry";
export class AgentLifecycleConflict extends Error {
constructor(message: string) { super(message); this.name = "AgentLifecycleConflict"; }
}
export function canConfigureAgentConnection(agent: { lifecycleState: string; status: string }): boolean {
return agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying"
|| (agent.lifecycleState === "ready" && !["paused", "terminated", "pending_approval"].includes(agent.status));
}
export function isAgentAwaitingSetup(agent: { lifecycleState: string; lifecycleHolds: readonly string[] }): boolean {
return ["preparing", "verifying", "resuming"].includes(agent.lifecycleState) && agent.lifecycleHolds.length === 0;
}
export function transition(state: AgentLifecycleState, command: LifecycleCommand,
resumeState: "preparing" | "verifying" | "ready" = "ready"): AgentLifecycleState {
if (command === "retry") {
if (["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state)) return state;
} else if (command === "terminate") {
if (["terminated", "rejected", "terminating", "cleaning_up"].includes(state)) return state;
return "terminating";
} else if (command === "approve") {
if (state === "pending_approval") return "preparing";
} else if (command === "reject") {
if (state === "pending_approval") return "rejected";
if (state === "rejected") return state;
} else if (command === "pause") {
if (state === "paused" || state === "pausing") return state;
if (["ready", "preparing", "verifying", "resuming"].includes(state)) return "pausing";
} else if (command === "resume") {
if (state === "ready" || state === "resuming") return state;
if (state === "paused") return "resuming";
} else if (command === "complete") {
switch (state) {
case "preparing": return "verifying";
case "verifying": return "ready";
case "pausing": return "paused";
case "resuming": return resumeState;
case "terminating": return "cleaning_up";
case "cleaning_up": return "terminated";
}
}
throw new AgentLifecycleConflict(`Cannot ${command} an agent in state ${state}`);
}
export function compatibilityStatus(state: AgentLifecycleState, executionStatus = "idle"): string {
if (state === "ready") return ["active", "idle", "running", "error"].includes(executionStatus) ? executionStatus : "idle";
if (state === "pending_approval") return "pending_approval";
if (["terminating", "cleaning_up", "terminated", "rejected"].includes(state)) return "terminated";
return "paused";
}
export function isLifecycleWorkPending(state: AgentLifecycleState): boolean {
return ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state);
}
export function assertAgentPurgeAllowed(agent: { lifecycleState: string }) {
if (!["terminated", "rejected"].includes(agent.lifecycleState)) {
throw new AgentLifecycleConflict("Complete termination before deleting the agent");
}
}
+144
View File
@@ -0,0 +1,144 @@
import { hireApprovalService as approvalRecords, type HireDecisionTarget } from "./adapters/approvals.js";
import { agents } from "@paperclipai/db";
import { and, asc, eq, gt, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import type { LifecycleEffects } from "./adapters/effects.js";
export type { LifecycleEffects } from "./adapters/effects.js";
import { updateAgentConfiguration } from "./adapters/configuration.js";
import type { AgentConfigurationPatch, UpdateAgentOptions } from "../../lib/agent-records.js";
import { purgeAgent } from "./adapters/deletion.js";
import { agentRecords } from "./adapters/records.js";
import { assertRootDatabase, createLifecycleStore } from "./adapters/postgres.js";
import { createLifecycleWorker } from "./application/worker.js";
import type { LifecycleDriver } from "./application/ports.js";
export { AgentLifecycleConflict, canConfigureAgentConnection, isAgentAwaitingSetup } from "./domain/policy.js";
export type { HireDecisionTarget } from "./adapters/approvals.js";
export type { LifecycleDriver, LifecycleAgent, LifecycleFailure } from "./application/ports.js";
const policyStates = ["preparing", "verifying", "ready", "pausing", "paused", "resuming"] as const;
const policyPageSize = 100;
const workers = new WeakMap<Db, ReturnType<typeof createLifecycleWorker>>();
export function configureAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun = () => true) {
assertRootDatabase(db);
const worker = createLifecycleWorker(createLifecycleStore(db, effects), driver, effects.reportFailure, canRun);
workers.set(db, worker);
return worker;
}
export function scheduleAgentLifecycle(db: Db, id: string) {
// The periodic sweep retries work if the process stops before this call.
return workers.get(db)?.process(id).catch(() => {});
}
export function createAgentLifecycle(db: Db, effects: LifecycleEffects) {
assertRootDatabase(db);
const records = agentRecords(db, effects);
const store = createLifecycleStore(db, effects);
async function change(id: string, command: "pause" | "resume" | "terminate" | "retry", reason?: string) {
const result = await store.change(id, command, { reason });
if (result) scheduleAgentLifecycle(db, id);
return result ? records.getById(id) : null;
}
async function decideHire(target: HireDecisionTarget, decision: "approved" | "rejected", userId: string, note?: string | null) {
const result = await approvalRecords(db, effects).decide(target, decision, userId, note);
if (result?.hireApprovedAgentId) scheduleAgentLifecycle(db, result.hireApprovedAgentId);
return result;
}
return {
async get(id: string) {
const agent = await store.get(id);
return agent ? {
id: agent.id, companyId: agent.companyId, lifecycleState: agent.lifecycleState,
lifecycleVersion: agent.lifecycleVersion, lifecycleError: agent.lifecycleError,
} : null;
},
async requestHire(...args: Parameters<typeof records.create>) {
const agent = await records.create(...args);
scheduleAgentLifecycle(db, agent.id);
return agent;
},
approveHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "approved", userId, note),
rejectHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "rejected", userId, note),
async updateConfiguration(id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions) {
const existing = await records.getById(id);
if (!existing) return null;
const agent = await effects.transaction(db, existing.companyId, (tx, publications) =>
updateAgentConfiguration(tx, effects, id, data, options, publications));
if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id);
scheduleAgentLifecycle(db, id);
return agent;
},
async updateAndTransition(id: string, command: "pause" | "resume" | "terminate", data: AgentConfigurationPatch, options?: UpdateAgentOptions) {
const existing = await records.getById(id);
if (!existing) return null;
const agent = await effects.transaction(db, existing.companyId, async (tx, publications) => {
await updateAgentConfiguration(tx, effects, id, data, options, publications);
await createLifecycleStore(tx, effects).change(id, command, { reason: command === "resume" ? "user" : "manual" });
return agentRecords(tx, effects).getById(id);
});
if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id);
scheduleAgentLifecycle(db, id);
return agent;
},
async reconcilePolicyHolds(companyId: string, agentId?: string | null) {
if (agentId === null) return;
const rows = await db.select({ id: agents.id }).from(agents).where(and(
eq(agents.companyId, companyId), agentId ? eq(agents.id, agentId) : undefined,
inArray(agents.lifecycleState, policyStates),
));
for (const row of rows) {
await store.change(row.id, "reconcile");
scheduleAgentLifecycle(db, row.id);
}
},
pauseAgent: (id: string, reason = "manual") => change(id, "pause", reason),
resumeAgent: (id: string, reason = "user") => change(id, "resume", reason),
terminateAgent: (id: string) => change(id, "terminate"),
retry: (id: string) => change(id, "retry"),
clearError: records.clearError,
purgeAgent: (id: string) => purgeAgent(db, effects, id),
};
}
export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun: () => boolean) {
const worker = configureAgentLifecycle(db, effects, driver, canRun);
let running: Promise<void> | undefined;
let stopped = false;
let nextPolicySweep = 0;
let policyCursor: string | undefined;
const store = createLifecycleStore(db, effects);
function sweep() {
if (stopped || !canRun()) return Promise.resolve();
return running ??= (async () => {
if (Date.now() >= nextPolicySweep) {
try {
const rows = await db.select({ id: agents.id }).from(agents).where(and(
inArray(agents.lifecycleState, policyStates), policyCursor ? gt(agents.id, policyCursor) : undefined,
)).orderBy(asc(agents.id)).limit(policyPageSize);
for (const row of rows) {
await store.change(row.id, "reconcile");
scheduleAgentLifecycle(db, row.id);
policyCursor = row.id;
}
if (rows.length < policyPageSize) policyCursor = undefined;
}
catch (error) { effects.reportFailure({ stage: "policy_scan" }, error); throw error; }
nextPolicySweep = Date.now() + 60_000;
}
await worker.sweep();
})().finally(() => { running = undefined; });
}
const timer = setInterval(() => { void sweep().catch(() => {}); }, 5_000);
timer.unref?.();
return {
sweep,
async stop() {
stopped = true;
clearInterval(timer);
workers.delete(db);
try { await running; } finally { await worker.stop(); }
},
};
}