mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 11:01:16 +01:00
refactor(server): move the release half of the deferred wake state machine into a wake-queue module (#13132)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The heartbeat service releases deferred issue wakes and promotes the next run > - The release logic sat in a large service function, which made its decisions and database effects hard to test > - A promotion race could reopen an issue without creating the promoted run > - Company checks did not protect every read and write, and some readers used a second transaction connection > - This pull request moves the release logic into a layered wake-queue module and closes these race and company-scope defects > - The benefit is clearer decisions, safer writes, and focused tests while existing callers keep the same entry point ## Linked Issues or Issue Description Refs: #10195 ## What Changed - Move the release half of deferred issue execution from `heartbeat.ts` into `server/src/modules/wake-queue/`. - Add pure policy decisions with table-driven tests. - Claim a wake before the reopen write and advance to the next wake when the claim fails. - Add company predicates to guarded reads and writes. - Pass the transaction-scoped issue snapshot to reader ports. - Keep `releaseIssueExecutionAndPromote` as the public wrapper. ## Verification - Run `pnpm check:module-boundaries`. - Run `pnpm exec tsc --noEmit` inside `server/` and compare the result with the known baseline. - Run the wake-queue unit and adapter tests in continuous integration. - Check the promotion-claim ordering, guarded writes, transaction-scoped reads, and cross-company outcomes. - Search the pull request diff for internal issue identifiers. ## Risks - The refactor changes the transaction path for deferred wake release. - A stale or lost wake claim now skips that wake and continues with the next queued wake. - The public wrapper keeps its name and signature, which limits caller risk. - Continuous integration must confirm the full server test suite and build. ## Model Used OpenAI Codex, GPT-5, exact runtime model version not exposed, context window not exposed, with shell, Git, and GitHub tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
018ca5daaf
commit
6dd48cad43
14 files changed
+3896
-1574
No files matched your search
@@ -0,0 +1,439 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createPostgresWakeQueueAdapter } from "./postgres.js";
|
||||
import type { WakeQueuePostgresAdapterDeps } from "./postgres.js";
|
||||
|
||||
// Proves the atomicity and company-scope properties the security review
|
||||
// requires: every mutation names `companyId` in its own SQL `WHERE` clause,
|
||||
// a foreign-company row is invisible to a read, and a deferred-status
|
||||
// compare-and-set that affects no row leaves no other trace. The decision
|
||||
// branching itself is proven against plain facts in `domain/policy.test.ts`.
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres wake-queue adapter tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
const stubDeps: WakeQueuePostgresAdapterDeps = {
|
||||
resolveResponsibleUserId: async () => "responsible-user",
|
||||
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
|
||||
resolveSessionBeforeForWakeup: async () => null,
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issueComments);
|
||||
// `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`,
|
||||
// so the run row must go first.
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompany(): Promise<string> {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
return companyId;
|
||||
}
|
||||
|
||||
async function seedAgent(input: { companyId: string; name?: string }): Promise<string> {
|
||||
const agentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId: input.companyId,
|
||||
name: input.name ?? "CodexCoder",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function seedIssue(input: {
|
||||
companyId: string;
|
||||
issueId?: string;
|
||||
status?: string;
|
||||
assigneeAgentId?: string | null;
|
||||
executionRunId?: string | null;
|
||||
checkoutRunId?: string | null;
|
||||
}): Promise<string> {
|
||||
const issueId = input.issueId ?? randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId: input.companyId,
|
||||
title: "Wake-queue adapter fixture issue",
|
||||
status: input.status ?? "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assigneeAgentId ?? null,
|
||||
executionRunId: input.executionRunId ?? null,
|
||||
checkoutRunId: input.checkoutRunId ?? null,
|
||||
});
|
||||
return issueId;
|
||||
}
|
||||
|
||||
async function seedRun(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
status?: string;
|
||||
contextSnapshot?: Record<string, unknown>;
|
||||
errorCode?: string | null;
|
||||
runtimeMode?: string;
|
||||
}): Promise<string> {
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
invocationSource: "on_demand",
|
||||
status: input.status ?? "failed",
|
||||
contextSnapshot: input.contextSnapshot ?? {},
|
||||
errorCode: input.errorCode ?? null,
|
||||
});
|
||||
return runId;
|
||||
}
|
||||
|
||||
async function seedDeferredWake(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
requestedByActorType?: string;
|
||||
requestedByActorId?: string | null;
|
||||
payload?: Record<string, unknown>;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
id,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: "automation",
|
||||
reason: "issue_commented",
|
||||
status: "deferred_issue_execution",
|
||||
requestedByActorType: input.requestedByActorType ?? "user",
|
||||
requestedByActorId: input.requestedByActorId ?? null,
|
||||
payload: { issueId: input.issueId, ...(input.payload ?? {}) },
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
// Review test (a): a foreign-company agent id produces the current failed
|
||||
// wake status and the current error text, and creates no run.
|
||||
it("fails a deferred wake whose agent belongs to a different company, without creating a run", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const foreignAgentId = await seedAgent({ companyId: otherCompanyId });
|
||||
const finishingAgentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: finishingAgentId, status: "in_progress" });
|
||||
// A finishing run status other than the legacy-reconciliation set (failed,
|
||||
// timed_out, interrupted, cancelled) reaches the module's own drain logic.
|
||||
const runId = await seedRun({ companyId, agentId: finishingAgentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId: foreignAgentId, issueId });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
|
||||
const candidate = await ports.writer.claimNextDeferredWake({ companyId, issueId: locked.primaryIssue.id });
|
||||
expect(candidate?.id).toBe(wakeId);
|
||||
const agent = await ports.reader.findInvokableAgent({ companyId, agentId: foreignAgentId });
|
||||
expect(agent).toBeNull();
|
||||
const failed = await ports.writer.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() });
|
||||
expect(failed).toBe(true);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("failed");
|
||||
expect(wakeRow?.error).toBe("Deferred wake could not be promoted: agent is not invokable");
|
||||
expect(wakeRow?.runId).toBeNull();
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]!.id).toBe(runId);
|
||||
});
|
||||
|
||||
// Review test (b): each release adapter mutation with a foreign company
|
||||
// affects no row.
|
||||
it("scopes every release mutation to its own company and affects no row across a company boundary", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
await adapter.withIssueExecutionLock(
|
||||
{ companyId, runId, now: new Date() },
|
||||
async (_locked, ports) => {
|
||||
const cancelledUnderWrongCompany = await ports.writer.cancelDeferredWake({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
reason: "cross-company cancel attempt",
|
||||
now: new Date(),
|
||||
});
|
||||
expect(cancelledUnderWrongCompany).toBe(false);
|
||||
|
||||
const failedUnderWrongCompany = await ports.writer.failDeferredWake({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
now: new Date(),
|
||||
});
|
||||
expect(failedUnderWrongCompany).toBe(false);
|
||||
|
||||
const normalizedUnderWrongCompany = await ports.writer.normalizeDeferredWakeCommentIds({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
payload: { issueId },
|
||||
liveCommentIds: ["c1"],
|
||||
now: new Date(),
|
||||
});
|
||||
expect(normalizedUnderWrongCompany).toBeNull();
|
||||
|
||||
const reopenedUnderWrongCompany = await ports.writer.reopenIssue({
|
||||
companyId: otherCompanyId,
|
||||
issueId,
|
||||
runId,
|
||||
});
|
||||
expect(reopenedUnderWrongCompany).toBeNull();
|
||||
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
},
|
||||
);
|
||||
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("deferred_issue_execution");
|
||||
expect(wakeRow?.error).toBeNull();
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("blocked");
|
||||
expect(issueRow?.executionState).toEqual({ phase: "running" });
|
||||
});
|
||||
|
||||
// Review defect: the reopen path must carry the company into every read,
|
||||
// lock, and write. A check before the write is not a boundary, because
|
||||
// `issues.company_id` can change between that check and the write.
|
||||
it("refuses to reopen an issue for a company that does not own it, and leaves the issue untouched", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
|
||||
reopened: null,
|
||||
};
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
captured.reopened = await ports.writer.reopenIssue({ companyId: otherCompanyId, issueId, runId });
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(captured.reopened).toBeNull();
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("blocked");
|
||||
expect(issueRow?.executionState).toEqual({ phase: "running" });
|
||||
});
|
||||
|
||||
it("reopens an issue for the company that owns it, and clears the execution state", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
|
||||
reopened: null,
|
||||
};
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
captured.reopened = await ports.writer.reopenIssue({ companyId, issueId, runId });
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(captured.reopened?.status).toBe("todo");
|
||||
expect(captured.reopened?.executionState).toBeNull();
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("todo");
|
||||
expect(issueRow?.executionState).toBeNull();
|
||||
});
|
||||
|
||||
// Review test (c): a deferred-status compare-and-set that affects no row
|
||||
// claims nothing, and no other write in the promotion path ever runs.
|
||||
it("fails the promotion claim when the deferred-status compare-and-set loses the race, before any other write", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
// A concurrent finalization already claimed this wake before the promotion claim runs.
|
||||
await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, wakeId));
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
const claimed = await ports.writer.claimDeferredWakeForPromotion({ companyId, wakeId, now: new Date() });
|
||||
expect(claimed).toBe(false);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]!.id).toBe(runId);
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.executionRunId).toBeNull();
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("cancelled");
|
||||
});
|
||||
|
||||
// `finalizePromotedWake`'s own writes guard against clobbering state a
|
||||
// concurrent write already changed: it never takes the issue's execution
|
||||
// lock away from a run that already holds it, and it never overwrites a
|
||||
// `runId` a wake row already carries. Both guards only matter as defense
|
||||
// in depth today (the release drain calls this at most once per
|
||||
// transaction), so this drives the port directly to prove the SQL itself,
|
||||
// independent of that call pattern.
|
||||
it("guards finalizePromotedWake's own writes against clobbering an already-set execution lock or runId", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const wakeIdA = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const wakeIdB = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
const deferredAgent = { id: agentId, companyId, name: "CodexCoder", invokable: true };
|
||||
|
||||
const finalizedRunIds: string[] = [];
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
|
||||
const finalize = async (wakeId: string) => {
|
||||
const promoted = await ports.writer.finalizePromotedWake({
|
||||
companyId,
|
||||
wakeId,
|
||||
deferredAgent,
|
||||
issue: locked.primaryIssue,
|
||||
finishingRun: locked.run,
|
||||
contextSnapshot: { issueId },
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: {},
|
||||
responsibleUserId: "responsible-user",
|
||||
sessionBefore: null,
|
||||
now: new Date(),
|
||||
});
|
||||
finalizedRunIds.push(promoted.id);
|
||||
};
|
||||
// The issue's execution lock is free; this call takes it.
|
||||
await finalize(wakeIdA);
|
||||
// The issue's execution lock is already held by the first call's run,
|
||||
// so this call's issue-lock write must no-op even though a run is
|
||||
// still created.
|
||||
await finalize(wakeIdB);
|
||||
// Repeating the same wake must not overwrite its now-set `runId`.
|
||||
await finalize(wakeIdA);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
const [runA, runB, runC] = finalizedRunIds;
|
||||
|
||||
const wakeRowA = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdA)))[0];
|
||||
const wakeRowB = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdB)))[0];
|
||||
expect(wakeRowA?.runId).toBe(runA);
|
||||
expect(wakeRowB?.runId).toBe(runB);
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.executionRunId).toBe(runA);
|
||||
expect(issueRow?.executionRunId).not.toBe(runB);
|
||||
expect(issueRow?.executionRunId).not.toBe(runC);
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
// All three finalize calls each still insert their own run row.
|
||||
expect(runs.map((run) => run.id).sort()).toEqual([runId, runA, runB, runC].sort());
|
||||
});
|
||||
|
||||
it("locks the context issue and every sibling issue in id order, and two concurrent releases do not deadlock", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueA = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const issueB = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const runA = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueA } });
|
||||
const runB = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueB } });
|
||||
await db.update(issues).set({ executionRunId: runA, checkoutRunId: runB }).where(eq(issues.id, issueA));
|
||||
await db.update(issues).set({ executionRunId: runB, checkoutRunId: runA }).where(eq(issues.id, issueB));
|
||||
|
||||
const adapterA = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const adapterB = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const releaseA = adapterA.withIssueExecutionLock({ companyId, runId: runA, now: new Date() }, async (_locked, _ports) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
const releaseB = adapterB.withIssueExecutionLock({ companyId, runId: runB, now: new Date() }, async (_locked, _ports) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
|
||||
await expect(Promise.all([releaseA, releaseB])).resolves.toBeDefined();
|
||||
|
||||
const rows = await db.select().from(issues).where(eq(issues.companyId, companyId));
|
||||
for (const row of rows) {
|
||||
expect(row.executionRunId).toBeNull();
|
||||
expect(row.checkoutRunId).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("clears both lock columns on every sibling and keeps a transferred executionRunId", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const finishingRunId = await seedRun({ companyId, agentId, contextSnapshot: {} });
|
||||
const retryRunId = await seedRun({ companyId, agentId, contextSnapshot: {}, status: "queued" });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, executionRunId: retryRunId, checkoutRunId: finishingRunId });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId: finishingRunId, now: new Date() }, async () => ({
|
||||
outcome: { kind: "released" as const },
|
||||
postCommitEffects: [],
|
||||
}));
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
// executionRunId already pointed at the retry, not the finishing run, so it must survive.
|
||||
expect(issueRow?.executionRunId).toBe(retryRunId);
|
||||
expect(issueRow?.checkoutRunId).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,817 @@
|
||||
import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
} from "@paperclipai/db";
|
||||
import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js";
|
||||
import { evaluateAgentInvokability } from "../../../services/agent-invokability.js";
|
||||
import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js";
|
||||
import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js";
|
||||
import { issueService } from "../../../services/issues.js";
|
||||
import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
|
||||
import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js";
|
||||
import { withRecoveryContext } from "../../../services/recovery/status-only-context.js";
|
||||
import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js";
|
||||
import {
|
||||
buildConfigurationIncompleteRecoveryNoticeSeed,
|
||||
buildExecutionReviewParticipantRecoveryNoticeSeed,
|
||||
buildImmediateExecutionPathRecoveryNoticeSeed,
|
||||
buildWorkspaceValidationRecoveryNoticeSeed,
|
||||
} from "../../../services/recovery/stranded-notice.js";
|
||||
import {
|
||||
queuedCommentIdsFromWakePayload,
|
||||
withQueuedCommentIdsInWakePayload,
|
||||
} from "../../../services/issue-queued-comment-queue.js";
|
||||
import { extractWakeCommentIds } from "../../run-dispatch/index.js";
|
||||
import { hasInteractionContinuationWakeContext } from "../domain/context.js";
|
||||
import type {
|
||||
DeferredWakeCandidate,
|
||||
InvokableAgentSnapshot,
|
||||
IssueLockWriter,
|
||||
IssueSnapshot,
|
||||
LockedIssueExecution,
|
||||
ReleaseTransactionResult,
|
||||
RunSnapshot,
|
||||
WakeQueueReader,
|
||||
WakeQueueWriter,
|
||||
} from "../application/ports.js";
|
||||
import type { RunSummary } from "../application/types.js";
|
||||
import { WakeQueueApplicationError } from "../application/types.js";
|
||||
|
||||
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
|
||||
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
|
||||
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
|
||||
const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
|
||||
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
|
||||
const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
|
||||
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = typeof issues.$inferSelect;
|
||||
|
||||
function parseObject(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function readNonEmptyString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function normalizeAgentNameKey(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : null;
|
||||
}
|
||||
|
||||
function isWorkspaceValidationFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
|
||||
return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
|
||||
}
|
||||
|
||||
function isConfigurationIncompleteFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
|
||||
return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
|
||||
}
|
||||
|
||||
function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
|
||||
const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
status: row.status,
|
||||
runtimeMode: row.runtimeMode,
|
||||
errorCode: row.errorCode,
|
||||
responsibleUserId: row.responsibleUserId,
|
||||
contextSnapshot: parseObject(row.contextSnapshot),
|
||||
configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null,
|
||||
};
|
||||
}
|
||||
|
||||
function toIssueSnapshot(row: IssueRow): IssueSnapshot {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
identifier: row.identifier ?? "",
|
||||
status: row.status,
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
hiddenAt: row.hiddenAt,
|
||||
originKind: row.originKind,
|
||||
monitorNextCheckAt: row.monitorNextCheckAt,
|
||||
executionState: (row.executionState as Record<string, unknown> | null) ?? null,
|
||||
responsibleUserId: row.responsibleUserId,
|
||||
parentId: row.parentId,
|
||||
originId: row.originId,
|
||||
originRunId: row.originRunId,
|
||||
};
|
||||
}
|
||||
|
||||
function toRunSummary(row: HeartbeatRunRow): RunSummary {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
invocationSource: row.invocationSource,
|
||||
triggerDetail: row.triggerDetail,
|
||||
wakeupRequestId: row.wakeupRequestId,
|
||||
};
|
||||
}
|
||||
|
||||
function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate {
|
||||
const payload = parseObject(row.payload);
|
||||
const queuedCommentIds = queuedCommentIdsFromWakePayload(payload);
|
||||
const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
|
||||
const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
|
||||
const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason);
|
||||
const queuedReason = wakeReason ?? readNonEmptyString(row.reason);
|
||||
const queuedWakeIsCommentOnly =
|
||||
!queuedReason ||
|
||||
queuedReason === "issue_commented" ||
|
||||
queuedReason === "issue_reopened_via_comment" ||
|
||||
queuedReason === "issue_comment_mentioned";
|
||||
const preservesIndependentContinuation =
|
||||
hasInteractionContinuationWakeContext(deferredContextSeed) ||
|
||||
deferredContextSeed.resumeIntent === true ||
|
||||
!queuedWakeIsCommentOnly;
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
reason: row.reason,
|
||||
source: row.source,
|
||||
triggerDetail: row.triggerDetail,
|
||||
requestedByActorType: row.requestedByActorType,
|
||||
requestedByActorId: row.requestedByActorId,
|
||||
payload,
|
||||
queuedCommentIds,
|
||||
preservesIndependentContinuation,
|
||||
deferredContextSeed,
|
||||
deferredCommentIds,
|
||||
wakeReason,
|
||||
};
|
||||
}
|
||||
|
||||
export type WakeQueuePostgresAdapterDeps = {
|
||||
resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
|
||||
getRoutineEnv: WakeQueueReader["getRoutineEnv"];
|
||||
resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
|
||||
};
|
||||
|
||||
function buildReader(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueReader {
|
||||
return {
|
||||
async findInvokableAgent({ companyId, agentId }): Promise<InvokableAgentSnapshot | null> {
|
||||
const agent = await tx
|
||||
.select()
|
||||
.from(agents)
|
||||
.where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!agent) return null;
|
||||
const companyAgents = await tx
|
||||
.select({ id: agents.id, companyId: agents.companyId, name: agents.name, reportsTo: agents.reportsTo, status: agents.status })
|
||||
.from(agents)
|
||||
.where(eq(agents.companyId, companyId));
|
||||
const invokability = evaluateAgentInvokability(agent, companyAgents);
|
||||
return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable };
|
||||
},
|
||||
resolveResponsibleUserId: deps.resolveResponsibleUserId,
|
||||
getRoutineEnv: deps.getRoutineEnv,
|
||||
resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
|
||||
};
|
||||
}
|
||||
|
||||
function buildWriter(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueWriter {
|
||||
const treeControlSvc = issueTreeControlService(tx);
|
||||
const issuesSvc = issueService(tx);
|
||||
|
||||
return {
|
||||
async claimNextDeferredWake({ companyId, issueId }) {
|
||||
const row = await tx
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(agentWakeupRequests.requestedAt))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row ? toDeferredWakeCandidate(row) : null;
|
||||
},
|
||||
|
||||
async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) {
|
||||
const rows = await tx
|
||||
.select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId })
|
||||
.from(issueComments)
|
||||
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queuedCommentIds)));
|
||||
const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId;
|
||||
const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => {
|
||||
const row = rows.find((candidate) => candidate.id === commentId);
|
||||
return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId));
|
||||
});
|
||||
const containedSelfAuthoredComment = rows.some(
|
||||
(row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId,
|
||||
);
|
||||
return { liveNonSelfCommentIds, containedSelfAuthoredComment };
|
||||
},
|
||||
|
||||
async cancelDeferredWake({ companyId, wakeId, reason, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
|
||||
async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
const row = rows[0];
|
||||
return row ? toDeferredWakeCandidate(row) : null;
|
||||
},
|
||||
|
||||
async failDeferredWake({ companyId, wakeId, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "failed",
|
||||
finishedAt: now,
|
||||
error: "Deferred wake could not be promoted: agent is not invokable",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
|
||||
async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) {
|
||||
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId);
|
||||
if (!activePauseHold) {
|
||||
return {
|
||||
activePauseHold: false,
|
||||
treeHoldInteractionWake: false,
|
||||
holdId: null,
|
||||
rootIssueId: null,
|
||||
mode: null,
|
||||
reason: null,
|
||||
releasePolicy: null,
|
||||
};
|
||||
}
|
||||
const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, {
|
||||
companyId,
|
||||
issueId,
|
||||
agentId: wakeAgentId,
|
||||
contextSnapshot: deferredContextSeed,
|
||||
requestedByActorType,
|
||||
requestedByActorId,
|
||||
});
|
||||
return {
|
||||
activePauseHold: true,
|
||||
treeHoldInteractionWake,
|
||||
holdId: activePauseHold.holdId,
|
||||
rootIssueId: activePauseHold.rootIssueId,
|
||||
mode: activePauseHold.mode,
|
||||
reason: activePauseHold.reason,
|
||||
releasePolicy: activePauseHold.releasePolicy,
|
||||
};
|
||||
},
|
||||
|
||||
async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) {
|
||||
const rows = await tx
|
||||
.select({ createdByRunId: issueComments.createdByRunId })
|
||||
.from(issueComments)
|
||||
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds)));
|
||||
return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) };
|
||||
},
|
||||
|
||||
async reopenIssue({ companyId, issueId }) {
|
||||
const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx);
|
||||
return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null;
|
||||
},
|
||||
|
||||
async claimDeferredWakeForPromotion({ companyId, wakeId, now }) {
|
||||
const claimed = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "queued",
|
||||
reason: "issue_execution_promoted",
|
||||
claimedAt: null,
|
||||
finishedAt: null,
|
||||
error: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return claimed.length > 0;
|
||||
},
|
||||
|
||||
async finalizePromotedWake(input) {
|
||||
const newRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId: input.deferredAgent.companyId,
|
||||
agentId: input.deferredAgent.id,
|
||||
invocationSource: input.source,
|
||||
triggerDetail: input.triggerDetail,
|
||||
status: "queued",
|
||||
wakeupRequestId: input.wakeId,
|
||||
contextSnapshot: input.contextSnapshot,
|
||||
responsibleUserId: input.responsibleUserId,
|
||||
sessionIdBefore: input.sessionBefore,
|
||||
continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt),
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
// `claimDeferredWakeForPromotion` already moved this row off
|
||||
// `deferred_issue_execution` inside this same transaction, so no
|
||||
// concurrent claimer can still match that guard; this extra `runId is
|
||||
// null` guard only protects against writing the link twice.
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: newRun.id, updatedAt: input.now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, input.wakeId),
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
isNull(agentWakeupRequests.runId),
|
||||
),
|
||||
);
|
||||
|
||||
// Promoted mention wakes are issue-scoped, not issue ownership
|
||||
// transfers. The lock-clearing step earlier in this transaction
|
||||
// already set `executionRunId` to null for this issue, so the `is
|
||||
// null` guard only protects against taking the lock twice.
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: newRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name),
|
||||
executionLockedAt: input.now,
|
||||
updatedAt: input.now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(issues.id, input.issue.id),
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.assigneeAgentId, input.deferredAgent.id),
|
||||
isNull(issues.executionRunId),
|
||||
),
|
||||
);
|
||||
|
||||
return toRunSummary(newRun);
|
||||
},
|
||||
|
||||
async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) {
|
||||
const row = await tx
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]),
|
||||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||||
sql`${heartbeatRuns.id} <> ${excludeRunId}`,
|
||||
agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row !== null;
|
||||
},
|
||||
|
||||
async hasExplicitBlockerPath({ companyId, issueId }) {
|
||||
const row = await tx
|
||||
.select({ issueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
eq(issues.companyId, companyId),
|
||||
notInArray(issues.status, ["done", "cancelled"]),
|
||||
isNull(issues.hiddenAt),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row !== null;
|
||||
},
|
||||
|
||||
async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) {
|
||||
return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc);
|
||||
},
|
||||
|
||||
async buildBlockedRecoveryNotice({ noticeKind, issueStatus, finishingRun }) {
|
||||
if (noticeKind === "workspace_validation") {
|
||||
return { notice: buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE };
|
||||
}
|
||||
if (noticeKind === "configuration_incomplete") {
|
||||
return {
|
||||
notice: buildConfigurationIncompleteRecoveryNoticeSeed(finishingRun.configurationIncompletePayload),
|
||||
recoveryCause: CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE,
|
||||
};
|
||||
}
|
||||
if (noticeKind === "execution_review_participant") {
|
||||
return {
|
||||
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
|
||||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
|
||||
};
|
||||
}
|
||||
return { notice: buildImmediateExecutionPathRecoveryNoticeSeed({ status: issueStatus }), recoveryCause: null };
|
||||
},
|
||||
|
||||
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
const executionState = parseIssueExecutionState(issue.executionState);
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
|
||||
payload: withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
retryOfRunId: finishingRun.id,
|
||||
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
|
||||
currentStageId: executionState?.currentStageId ?? null,
|
||||
currentStageType: executionState?.currentStageType ?? null,
|
||||
},
|
||||
"normal_model",
|
||||
),
|
||||
status: "queued",
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
|
||||
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
|
||||
source: "issue.execution_review_recovery",
|
||||
retryOfRunId: finishingRun.id,
|
||||
currentStageId: executionState?.currentStageId ?? null,
|
||||
currentStageType: executionState?.currentStageType ?? null,
|
||||
reviewRecoveryInstruction:
|
||||
"The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
|
||||
},
|
||||
"normal_model",
|
||||
),
|
||||
sessionIdBefore: sessionBefore,
|
||||
retryOfRunId: finishingRun.id,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: queuedRun.id, updatedAt: now })
|
||||
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
|
||||
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: queuedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
|
||||
executionLockedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
|
||||
|
||||
return toRunSummary(queuedRun);
|
||||
},
|
||||
|
||||
async queueImmediateRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
|
||||
const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed";
|
||||
const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery";
|
||||
const recoveryContextSnapshot = withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
wakeReason: recoveryReason,
|
||||
retryReason,
|
||||
source: recoverySource,
|
||||
retryOfRunId: finishingRun.id,
|
||||
},
|
||||
"normal_model",
|
||||
);
|
||||
|
||||
const routineEnvContext = await deps.getRoutineEnv({ companyId, issue });
|
||||
const responsibleUserId = await deps.resolveResponsibleUserId({
|
||||
companyId,
|
||||
contextSnapshot: recoveryContextSnapshot,
|
||||
issue,
|
||||
routineEnvContext,
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
existingRunResponsibleUserId: finishingRun.responsibleUserId,
|
||||
});
|
||||
if (!responsibleUserId) {
|
||||
throw new WakeQueueApplicationError(
|
||||
"responsible_user_unresolved",
|
||||
"Unable to resolve responsible user for recovery heartbeat run",
|
||||
{
|
||||
runId: finishingRun.id,
|
||||
agentId: recoveryAgent.id,
|
||||
companyId,
|
||||
issueId: issue.id,
|
||||
wakeReason: recoveryReason,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: recoveryReason,
|
||||
payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"),
|
||||
status: "queued",
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: recoveryContextSnapshot,
|
||||
responsibleUserId,
|
||||
sessionIdBefore: sessionBefore,
|
||||
retryOfRunId: finishingRun.id,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: queuedRun.id, updatedAt: now })
|
||||
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
|
||||
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: queuedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
|
||||
executionLockedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
|
||||
|
||||
return toRunSummary(queuedRun);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise<boolean> {
|
||||
const applies =
|
||||
run.runtimeMode === "native" &&
|
||||
["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
|
||||
issue.assigneeAgentId === run.agentId &&
|
||||
!["done", "cancelled"].includes(issue.status);
|
||||
if (!applies) return false;
|
||||
|
||||
const existing = await tx
|
||||
.select({ id: issueRecoveryActions.id })
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, issue.companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, issue.id),
|
||||
or(
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`,
|
||||
),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (!existing.length) {
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
phase: "terminal_failure",
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
nextAttemptAt: null,
|
||||
recoveryState: "blocked",
|
||||
failureCode: "native_continuation_requires_reconciliation",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(nativeRunFinalizations.companyId, issue.companyId),
|
||||
eq(nativeRunFinalizations.runId, run.id),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
),
|
||||
);
|
||||
await issueRecoveryActionService(tx).upsertSourceScoped({
|
||||
companyId: issue.companyId,
|
||||
sourceIssueId: issue.id,
|
||||
kind: "active_run_watchdog",
|
||||
ownerType: "board",
|
||||
returnOwnerAgentId: run.agentId,
|
||||
cause: "native_continuation_requires_reconciliation",
|
||||
fingerprint: `native-continuation:${run.id}`,
|
||||
evidence: { runId: run.id, originalFailureCode: run.errorCode },
|
||||
nextAction:
|
||||
"Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.",
|
||||
maxAttempts: 3,
|
||||
wakePolicy: null,
|
||||
supersedeOnIdentityChange: true,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter {
|
||||
return {
|
||||
async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
|
||||
return db.transaction(async (rawTx) => {
|
||||
const tx = rawTx as unknown as Db;
|
||||
const run = await tx
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!run) {
|
||||
throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`);
|
||||
}
|
||||
const runSnapshot = toRunSnapshot(run);
|
||||
const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
|
||||
// Lock the context issue (if any) and every issue that still references this
|
||||
// run, in id order, so two concurrent finalizations can never deadlock on
|
||||
// each other's row-lock acquisition order.
|
||||
await tx.execute(
|
||||
contextIssueId
|
||||
? sql`
|
||||
select id from issues
|
||||
where company_id = ${input.companyId}
|
||||
and (
|
||||
id = ${contextIssueId}
|
||||
or execution_run_id = ${run.id}
|
||||
or checkout_run_id = ${run.id}
|
||||
)
|
||||
order by id
|
||||
for update
|
||||
`
|
||||
: sql`
|
||||
select id from issues
|
||||
where company_id = ${input.companyId}
|
||||
and (execution_run_id = ${run.id} or checkout_run_id = ${run.id})
|
||||
order by id
|
||||
for update
|
||||
`,
|
||||
);
|
||||
|
||||
const candidateIssues = await tx
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
contextIssueId
|
||||
? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id))
|
||||
: or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(issues.id));
|
||||
|
||||
// Two separate updates: a retry can move `executionRunId` to a new run
|
||||
// while `checkoutRunId` still points at this one finishing.
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now })
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id)));
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ checkoutRunId: null, updatedAt: input.now })
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id)));
|
||||
|
||||
const issueRow =
|
||||
(contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null;
|
||||
|
||||
if (!issueRow || (issueRow.executionRunId && issueRow.executionRunId !== run.id)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
if (
|
||||
(isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) &&
|
||||
(issueRow.status === "todo" || issueRow.status === "in_progress") &&
|
||||
!issueRow.assigneeUserId &&
|
||||
issueRow.assigneeAgentId === run.agentId
|
||||
) {
|
||||
const configurationIncomplete = isConfigurationIncompleteFailedRun(run);
|
||||
const notice = configurationIncomplete
|
||||
? buildConfigurationIncompleteRecoveryNoticeSeed(runSnapshot.configurationIncompletePayload)
|
||||
: buildWorkspaceValidationRecoveryNoticeSeed();
|
||||
return {
|
||||
outcome: {
|
||||
kind: "blocked",
|
||||
issue: toIssueSnapshot(issueRow),
|
||||
previousStatus: issueRow.status as "todo" | "in_progress",
|
||||
notice,
|
||||
recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE : WORKSPACE_VALIDATION_RECOVERY_CAUSE,
|
||||
},
|
||||
postCommitEffects: [],
|
||||
run: runSnapshot,
|
||||
};
|
||||
}
|
||||
|
||||
if (legacyExecutionNeedsReconciliation(run)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
// An operator stop never promotes old queued work by itself. The next
|
||||
// explicit wake adopts those messages atomically when it queues a run.
|
||||
if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot };
|
||||
const result = await fn(locked, { reader: buildReader(tx, deps), writer: buildWriter(tx, deps) });
|
||||
return { ...result, run: runSnapshot };
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user