import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { authUsers, companyMemberships, activityLog, agents, companies, createDb, heartbeatRuns, issueComments, issueRelations, issues, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { errorHandler } from "../middleware/index.js"; import { issueRoutes } from "../routes/issues.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres stale execution lock route tests on this host: ${ embeddedPostgresSupport.reason ?? "unsupported environment" }`, ); } describeEmbeddedPostgres("stale issue execution lock routes", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-stale-execution-lock-routes-"); db = createDb(tempDb.connectionString); }, 20_000); afterEach(async () => { await db.delete(issueComments); await db.delete(issueRelations); await db.delete(activityLog); await db.delete(issues); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); await db.delete(authUsers); }); afterAll(async () => { await tempDb?.cleanup(); }); function createApp(actor: Express.Request["actor"]) { const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = actor; next(); }); app.use("/api", issueRoutes(db, {} as any)); app.use(errorHandler); return app; } async function seedCompanyAgentAndRuns() { const companyId = randomUUID(); const agentId = randomUUID(); const failedRunId = randomUUID(); const currentRunId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, }); await db.insert(authUsers).values({ id: "board-user", name: "Board", email: "board@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board-user", status: "active", membershipRole: "admin" }); await db.insert(agents).values({ id: agentId, companyId, name: "CodexCoder", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {}, }); await db.insert(heartbeatRuns).values([ { id: failedRunId, companyId, agentId, status: "failed", invocationSource: "manual", finishedAt: new Date(), }, { id: currentRunId, companyId, agentId, status: "running", invocationSource: "manual", startedAt: new Date(), }, ]); return { companyId, agentId, failedRunId, currentRunId }; } function agentActor(companyId: string, agentId: string, runId: string): Express.Request["actor"] { return { type: "agent", agentId, companyId, runId, source: "agent_jwt", }; } function boardActor(companyId: string): Express.Request["actor"] { return { type: "board", userId: "board-user", companyIds: [companyId], memberships: [{ companyId, membershipRole: "admin", status: "active" }], isInstanceAdmin: false, source: "session", }; } it("allows an assigned agent PATCH to recover a terminal stale executionRunId", async () => { const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Stale execution lock", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: null, executionRunId: failedRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); await db.update(heartbeatRuns) .set({ contextSnapshot: { issueId } }) .where(eq(heartbeatRuns.id, currentRunId)); const res = await request(createApp(agentActor(companyId, agentId, currentRunId))) .patch(`/api/issues/${issueId}`) .send({ title: "Recovered execution lock" }); expect(res.status, JSON.stringify(res.body)).toBe(200); expect(res.body.title).toBe("Recovered execution lock"); const row = await db .select({ title: issues.title, checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ title: "Recovered execution lock", checkoutRunId: currentRunId, executionRunId: currentRunId, }); }); it.each([ { status: "done" as const, title: "Done release preserves owner", completedAt: new Date(), retainAssignee: true }, { status: "cancelled" as const, title: "Cancelled release preserves owner", cancelledAt: new Date(), retainAssignee: true }, { status: "in_review" as const, title: "In review release preserves status" }, { status: "blocked" as const, title: "Blocked release preserves status" }, { status: "backlog" as const, title: "Backlog release relinquishes ownership" }, { status: "todo" as const, title: "Todo release relinquishes ownership" }, { status: "in_progress" as const, title: "Active release requeues work", expectedStatus: "todo" }, ])( "$title and clears execution locks", async ({ status, title, completedAt, cancelledAt, retainAssignee, expectedStatus = status }) => { const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title, status, priority: "medium", assigneeAgentId: agentId, checkoutRunId: currentRunId, executionRunId: currentRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), ...(completedAt ? { completedAt } : {}), ...(cancelledAt ? { cancelledAt } : {}), }); const res = await request(createApp(agentActor(companyId, agentId, currentRunId))) .post(`/api/issues/${issueId}/release`) .send(); expect(res.status, JSON.stringify(res.body)).toBe(200); const expectedAssignee = retainAssignee ? agentId : null; expect(res.body).toMatchObject({ status: expectedStatus, assigneeAgentId: expectedAssignee }); const row = await db .select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, executionAgentNameKey: issues.executionAgentNameKey, executionLockedAt: issues.executionLockedAt, completedAt: issues.completedAt, cancelledAt: issues.cancelledAt, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ status: expectedStatus, assigneeAgentId: expectedAssignee, checkoutRunId: null, executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, completedAt: completedAt ?? null, cancelledAt: cancelledAt ?? null, }); const events = await db.select().from(activityLog).where(eq(activityLog.entityId, issueId)); expect(events).toContainEqual(expect.objectContaining({ action: "issue.released", companyId, agentId, runId: currentRunId, })); if (retainAssignee) { // Repeated cleanup, including by the board, must not erase attribution. const retried = await request(createApp(boardActor(companyId))) .post(`/api/issues/${issueId}/release`) .send(); expect(retried.status, JSON.stringify(retried.body)).toBe(200); expect(retried.body).toMatchObject({ status, assigneeAgentId: agentId, checkoutRunId: null, executionRunId: null }); const [saved] = await db.select().from(issues).where(eq(issues.id, issueId)); expect(saved).toMatchObject(row); } }, ); it("allows the rightful assignee to release after the owning run failed", async () => { const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Failed run release", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: failedRunId, executionRunId: failedRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); const res = await request(createApp(agentActor(companyId, agentId, currentRunId))) .post(`/api/issues/${issueId}/release`) .send(); expect(res.status, JSON.stringify(res.body)).toBe(200); const row = await db .select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, executionLockedAt: issues.executionLockedAt, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ status: "todo", assigneeAgentId: null, checkoutRunId: null, executionRunId: null, executionLockedAt: null, }); }); it("lets the current assignee recover a timed_out stale checkout owner during PATCH", async () => { const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns(); const timedOutRunId = randomUUID(); const issueId = randomUUID(); await db.insert(heartbeatRuns).values({ id: timedOutRunId, companyId, agentId, status: "timed_out", invocationSource: "manual", finishedAt: new Date(), }); await db.insert(issues).values({ id: issueId, companyId, title: "Stale checkout lock", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: timedOutRunId, executionRunId: timedOutRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); await db.update(heartbeatRuns) .set({ contextSnapshot: { issueId } }) .where(eq(heartbeatRuns.id, currentRunId)); const res = await request(createApp(agentActor(companyId, agentId, currentRunId))) .patch(`/api/issues/${issueId}`) .send({ title: "Recovered stale checkout lock" }); expect(res.status, JSON.stringify(res.body)).toBe(200); const row = await db .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ checkoutRunId: currentRunId, executionRunId: currentRunId, }); }); it("still returns 409 when a different live checkout owner is active", async () => { const { companyId, agentId, failedRunId } = await seedCompanyAgentAndRuns(); const liveOwnerRunId = randomUUID(); const issueId = randomUUID(); await db.insert(heartbeatRuns).values({ id: liveOwnerRunId, companyId, agentId, status: "running", invocationSource: "manual", startedAt: new Date(), }); await db.insert(issues).values({ id: issueId, companyId, title: "Live checkout lock", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: liveOwnerRunId, executionRunId: liveOwnerRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); const res = await request(createApp(agentActor(companyId, agentId, failedRunId))) .patch(`/api/issues/${issueId}`) .send({ title: "Should fail" }); expect(res.status, JSON.stringify(res.body)).toBe(409); expect(res.body?.error).toBe("Issue run ownership conflict"); }); it("preserves live checkout ownership on checkout conflicts without retry side effects", async () => { const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns(); const contenderRunId = randomUUID(); const issueId = randomUUID(); await db.insert(heartbeatRuns).values({ id: contenderRunId, companyId, agentId, status: "running", invocationSource: "assignment", startedAt: new Date(), }); await db.insert(issues).values({ id: issueId, companyId, title: "Live checkout race", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: currentRunId, executionRunId: currentRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); const res = await request(createApp(agentActor(companyId, agentId, contenderRunId))) .post(`/api/issues/${issueId}/checkout`) .send({ agentId, expectedStatuses: ["todo", "backlog", "blocked", "in_review"], }); expect(res.status, JSON.stringify(res.body)).toBe(409); expect(res.body).toMatchObject({ error: "Issue checkout conflict", }); const row = await db .select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ status: "in_progress", assigneeAgentId: agentId, checkoutRunId: currentRunId, executionRunId: currentRunId, }); const checkoutActivity = await db .select() .from(activityLog) .where(eq(activityLog.action, "issue.checked_out")); expect(checkoutActivity).toHaveLength(0); }); it("restricts admin force-release to board users with company access and writes an audit event", async () => { const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Admin force release", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: currentRunId, executionRunId: failedRunId, executionAgentNameKey: "codexcoder", executionLockedAt: new Date(), }); await request(createApp(agentActor(companyId, agentId, currentRunId))) .post(`/api/issues/${issueId}/admin/force-release`) .expect(403); await request(createApp({ type: "board", userId: "outside-user", companyIds: [], memberships: [], isInstanceAdmin: false, source: "session", })) .post(`/api/issues/${issueId}/admin/force-release`) .expect(404); const res = await request(createApp(boardActor(companyId))) .post(`/api/issues/${issueId}/admin/force-release?clearAssignee=true`) .send(); expect(res.status, JSON.stringify(res.body)).toBe(200); expect(res.body.issue).toMatchObject({ id: issueId, assigneeAgentId: null, checkoutRunId: null, executionRunId: null, executionLockedAt: null, }); expect(res.body.previous).toEqual({ checkoutRunId: currentRunId, executionRunId: failedRunId, }); const audit = await db .select({ action: activityLog.action, actorType: activityLog.actorType, actorId: activityLog.actorId, details: activityLog.details, }) .from(activityLog) .where(eq(activityLog.action, "issue.admin_force_release")) .then((rows) => rows[0]); expect(audit).toMatchObject({ action: "issue.admin_force_release", actorType: "user", actorId: "board-user", details: { issueId, actorUserId: "board-user", prevCheckoutRunId: currentRunId, prevExecutionRunId: failedRunId, clearAssignee: true, }, }); }); it("self-heals a stale checkoutRunId via clearCheckoutRunIfTerminal on checkout (Fix B path)", async () => { // Reproduces the recurrence pattern: prior owning run died, executionRunId // was cleared by releaseIssueExecutionAndPromote, but checkoutRunId stayed // pinned to the dead run. The new agent's POST /checkout would 409 forever // without the clearCheckoutRunIfTerminal helper in svc.checkout. const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); const otherAgentId = randomUUID(); await db.insert(agents).values({ id: otherAgentId, companyId, name: "OtherAgent", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {}, }); await db.insert(issues).values({ id: issueId, companyId, title: "Stale checkout lock after reassignment", // Status off in_progress + checkoutRunId still set — adoptStaleCheckoutRun // cannot recover from this; only clearCheckoutRunIfTerminal can. status: "todo", priority: "high", assigneeAgentId: otherAgentId, checkoutRunId: failedRunId, executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, }); const res = await request(createApp(agentActor(companyId, otherAgentId, currentRunId))) .post(`/api/issues/${issueId}/checkout`) .send({ agentId: otherAgentId, expectedStatuses: ["todo", "backlog", "blocked", "in_review"], }); expect(res.status, JSON.stringify(res.body)).toBe(200); const row = await db .select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ status: "in_progress", assigneeAgentId: otherAgentId, checkoutRunId: currentRunId, executionRunId: currentRunId, }); }); });