fix(identity): read a provisioned agent key without an exclusive agent row lock (#21506)

Run setup reads an existing agent identity key under FOR KEY SHARE on the agent row instead of FOR UPDATE, so it no longer joins queued-run claim deadlocks. The lock still waits for an in-progress agent deletion. First provisioning uses the same lock with ON CONFLICT DO NOTHING.

Co-Authored-By: paperclipbugbot <paperclipbugbot@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-22 01:17:29 -08:00
1 parent 28ed2f1db0
commit 7671e0cf0d
2 files changed
+129 -21

No files matched your search

@@ -0,0 +1,107 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agents, companies, createDb, type Db } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentIdentityService } from "../services/agent-identity.js";
let db: Db;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let directory: string;
let companyId: string;
const gate = () => { let open!: () => void; const opened = new Promise<void>(resolve => { open = resolve; }); return { opened, open }; };
const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const lockWaiters = async (pattern: string) => (await db.execute<{ count: number }>(sql`select count(*)::int as count
from pg_stat_activity where datname = current_database() and wait_event_type = 'Lock' and query like ${pattern}`))[0]?.count;
const company = (issuePrefix: string) => db.insert(companies).values({ name: issuePrefix, issuePrefix, defaultResponsibleUserId: "identity-test-owner" }).returning();
const newAgent = async () => (await db.insert(agents).values({ companyId, name: randomUUID(), role: "engineer" }).returning())[0];
beforeAll(async () => {
directory = await mkdtemp(join(tmpdir(), "paperclip-identity-lock-"));
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key"));
database = await startEmbeddedPostgresTestDatabase("paperclip-identity-lock-");
db = createDb(database.connectionString);
companyId = (await company("KLO"))[0].id;
});
afterAll(async () => {
await database?.cleanup();
vi.unstubAllEnvs();
await rm(directory, { recursive: true, force: true });
});
it("run setup reuses a key without queuing behind run claims or joining their deadlock", async () => {
const agent = await newAgent();
const identity = await agentIdentityService(db).ensureAgentIdentity(companyId, agent.id);
const [other] = await company("KLK");
const lockOther = (tx: Db) => tx.select({ id: companies.id }).from(companies).where(eq(companies.id, other.id)).for("update");
const claimLocked = gate(), workerLocked = gate(), claimContinues = gate(), workerContinues = gate();
const claim = db.transaction(async tx => {
await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, agent.id)).for("share");
claimLocked.open(); await claimContinues.opened;
await lockOther(tx as unknown as Db);
});
await claimLocked.opened;
const worker = db.transaction(async tx => {
await lockOther(tx as unknown as Db);
workerLocked.open(); await workerContinues.opened;
await tx.update(agents).set({ name: randomUUID() }).where(eq(agents.id, agent.id));
});
await workerLocked.opened;
let setupSettled = false;
const setup = agentIdentityService(db).ensureAgentIdentity(companyId, agent.id).finally(() => { setupSettled = true; });
void setup.catch(() => undefined);
await vi.waitFor(async () => {
expect(setupSettled || await lockWaiters('select %"agents"%') === 1).toBe(true);
}, { timeout: 10_000, interval: 25 });
workerContinues.open();
await vi.waitFor(async () => expect(await lockWaiters('update "agents"%')).toBe(1), { timeout: 10_000, interval: 25 });
claimContinues.open();
const [setupResult, ...cycle] = await Promise.allSettled([setup, claim, worker]);
expect(setupResult).toEqual({ status: "fulfilled", value: identity });
expect(cycle.filter(result => result.status === "rejected")).toHaveLength(1);
}, 30_000);
it("run setup does not return the key of an agent whose deletion is in progress", async () => {
const agent = await newAgent();
await agentIdentityService(db).ensureAgentIdentity(companyId, agent.id);
const locked = gate(), commitDeletion = gate();
const deletion = db.transaction(async tx => {
await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, agent.id)).for("update");
locked.open(); await commitDeletion.opened;
await tx.delete(agents).where(eq(agents.id, agent.id));
});
await locked.opened;
let setupSettled = false;
const setup = agentIdentityService(db).ensureAgentIdentity(companyId, agent.id).finally(() => { setupSettled = true; });
void setup.catch(() => undefined);
await vi.waitFor(async () => {
expect(setupSettled || await lockWaiters('select %"agents"%') === 1).toBe(true);
}, { timeout: 10_000, interval: 25 });
expect(setupSettled).toBe(false);
commitDeletion.open();
await deletion;
await expect(setup).rejects.toMatchObject({ status: 404 });
}, 30_000);
it("first provisioning does not wait for a run claim's share lock", async () => {
const agent = await newAgent();
const holding = gate(), release = gate();
const claim = db.transaction(async tx => {
await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, agent.id)).for("share");
holding.open(); await release.opened;
});
await holding.opened;
try {
const provisioned = await Promise.race([agentIdentityService(db).ensureAgentIdentity(companyId, agent.id),
pause(5_000).then(() => { throw new Error("first provisioning waited for a share lock"); })]);
expect(provisioned.keyId).toMatch(/^sha256:/);
} finally {
release.open();
await claim;
}
expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toMatchObject({ keyId: expect.stringMatching(/^sha256:/) });
}, 30_000);
+22 -21
View File
@@ -42,30 +42,31 @@ export function agentIdentityService(db: Db) {
async ensureAgentIdentity(companyId: string, agentId: string) {
return db.transaction(async (tx) => {
// Lock the parent even when there is no identity row yet. This also
// serializes deletion and works inside the agent-creation transaction.
const [owner] = await tx.select({ id: agents.id }).from(agents)
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("update");
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("key share");
if (!owner) throw notFound("Agent not found");
let [row] = await tx.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
if (!row) {
const keys = generateKeyPairSync("ed25519");
const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString();
const keyId = fingerprint(keys.publicKey.export({ type: "spki", format: "der" }));
const privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString();
const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem });
[row] = await tx.insert(agentIdentityKeys).values({
companyId, agentId, algorithm: "Ed25519", keyId, publicKeyPem,
privateKeyMaterial: encrypted.material,
}).returning();
await persistActivity(tx as unknown as Db, {
companyId, actorType: "system", actorId: "system",
action: "agent.identity.created", entityType: "agent", entityId: agentId,
details: { algorithm: "Ed25519", keyId },
});
const [existing] = await tx.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
if (existing) return resolveIdentity(existing);
const keys = generateKeyPairSync("ed25519");
const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString();
const keyId = fingerprint(keys.publicKey.export({ type: "spki", format: "der" }));
const privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString();
const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem });
const [created] = await tx.insert(agentIdentityKeys).values({
companyId, agentId, algorithm: "Ed25519", keyId, publicKeyPem,
privateKeyMaterial: encrypted.material,
}).onConflictDoNothing({ target: agentIdentityKeys.agentId }).returning();
if (!created) {
const [winner] = await tx.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
if (!winner) throw new Error("Agent identity key belongs to another owner");
return resolveIdentity(winner);
}
// Fail closed on corruption or a wrong master key; never replace a key.
return resolveIdentity(row);
await persistActivity(tx as unknown as Db, {
companyId, actorType: "system", actorId: "system",
action: "agent.identity.created", entityType: "agent", entityId: agentId,
details: { algorithm: "Ed25519", keyId },
});
return resolveIdentity(created);
});
},
};