refactor(server): move the release half of the deferred wake state machine into a wake-queue module (#13132)

## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work
> - The heartbeat service releases deferred issue wakes and promotes the
next run
> - The release logic sat in a large service function, which made its
decisions and database effects hard to test
> - A promotion race could reopen an issue without creating the promoted
run
> - Company checks did not protect every read and write, and some
readers used a second transaction connection
> - This pull request moves the release logic into a layered wake-queue
module and closes these race and company-scope defects
> - The benefit is clearer decisions, safer writes, and focused tests
while existing callers keep the same entry point

## Linked Issues or Issue Description

Refs: #10195

## What Changed

- Move the release half of deferred issue execution from `heartbeat.ts`
into `server/src/modules/wake-queue/`.
- Add pure policy decisions with table-driven tests.
- Claim a wake before the reopen write and advance to the next wake when
the claim fails.
- Add company predicates to guarded reads and writes.
- Pass the transaction-scoped issue snapshot to reader ports.
- Keep `releaseIssueExecutionAndPromote` as the public wrapper.

## Verification

- Run `pnpm check:module-boundaries`.
- Run `pnpm exec tsc --noEmit` inside `server/` and compare the result
with the known baseline.
- Run the wake-queue unit and adapter tests in continuous integration.
- Check the promotion-claim ordering, guarded writes, transaction-scoped
reads, and cross-company outcomes.
- Search the pull request diff for internal issue identifiers.

## Risks

- The refactor changes the transaction path for deferred wake release.
- A stale or lost wake claim now skips that wake and continues with the
next queued wake.
- The public wrapper keeps its name and signature, which limits caller
risk.
- Continuous integration must confirm the full server test suite and
build.

## Model Used

OpenAI Codex, GPT-5, exact runtime model version not exposed, context
window not exposed, with shell, Git, and GitHub tool use.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip authored and GitHub committed 2026-09-09 21:01:26 -07:00
1 parent 018ca5daaf
commit 6dd48cad43
14 files changed
+3896 -1574

No files matched your search

@@ -0,0 +1,439 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
agentWakeupRequests,
agents,
companies,
createDb,
heartbeatRuns,
issueComments,
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "../../../__tests__/helpers/embedded-postgres.js";
import { createPostgresWakeQueueAdapter } from "./postgres.js";
import type { WakeQueuePostgresAdapterDeps } from "./postgres.js";
// Proves the atomicity and company-scope properties the security review
// requires: every mutation names `companyId` in its own SQL `WHERE` clause,
// a foreign-company row is invisible to a read, and a deferred-status
// compare-and-set that affects no row leaves no other trace. The decision
// branching itself is proven against plain facts in `domain/policy.test.ts`.
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres wake-queue adapter tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("wake-queue postgres adapter", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
const stubDeps: WakeQueuePostgresAdapterDeps = {
resolveResponsibleUserId: async () => "responsible-user",
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
resolveSessionBeforeForWakeup: async () => null,
};
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
await db.delete(issueComments);
// `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`,
// so the run row must go first.
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(issues);
await db.delete(agents);
await db.delete(companies);
});
afterAll(async () => {
await tempDb?.cleanup();
});
async function seedCompany(): Promise<string> {
const companyId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
defaultResponsibleUserId: "responsible-user",
});
return companyId;
}
async function seedAgent(input: { companyId: string; name?: string }): Promise<string> {
const agentId = randomUUID();
await db.insert(agents).values({
id: agentId,
companyId: input.companyId,
name: input.name ?? "CodexCoder",
role: "engineer",
status: "active",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
permissions: {},
});
return agentId;
}
async function seedIssue(input: {
companyId: string;
issueId?: string;
status?: string;
assigneeAgentId?: string | null;
executionRunId?: string | null;
checkoutRunId?: string | null;
}): Promise<string> {
const issueId = input.issueId ?? randomUUID();
await db.insert(issues).values({
id: issueId,
companyId: input.companyId,
title: "Wake-queue adapter fixture issue",
status: input.status ?? "in_progress",
priority: "medium",
assigneeAgentId: input.assigneeAgentId ?? null,
executionRunId: input.executionRunId ?? null,
checkoutRunId: input.checkoutRunId ?? null,
});
return issueId;
}
async function seedRun(input: {
companyId: string;
agentId: string;
status?: string;
contextSnapshot?: Record<string, unknown>;
errorCode?: string | null;
runtimeMode?: string;
}): Promise<string> {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({
id: runId,
companyId: input.companyId,
agentId: input.agentId,
invocationSource: "on_demand",
status: input.status ?? "failed",
contextSnapshot: input.contextSnapshot ?? {},
errorCode: input.errorCode ?? null,
});
return runId;
}
async function seedDeferredWake(input: {
companyId: string;
agentId: string;
issueId: string;
requestedByActorType?: string;
requestedByActorId?: string | null;
payload?: Record<string, unknown>;
}): Promise<string> {
const id = randomUUID();
await db.insert(agentWakeupRequests).values({
id,
companyId: input.companyId,
agentId: input.agentId,
source: "automation",
reason: "issue_commented",
status: "deferred_issue_execution",
requestedByActorType: input.requestedByActorType ?? "user",
requestedByActorId: input.requestedByActorId ?? null,
payload: { issueId: input.issueId, ...(input.payload ?? {}) },
});
return id;
}
// Review test (a): a foreign-company agent id produces the current failed
// wake status and the current error text, and creates no run.
it("fails a deferred wake whose agent belongs to a different company, without creating a run", async () => {
const companyId = await seedCompany();
const otherCompanyId = await seedCompany();
const foreignAgentId = await seedAgent({ companyId: otherCompanyId });
const finishingAgentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: finishingAgentId, status: "in_progress" });
// A finishing run status other than the legacy-reconciliation set (failed,
// timed_out, interrupted, cancelled) reaches the module's own drain logic.
const runId = await seedRun({ companyId, agentId: finishingAgentId, contextSnapshot: { issueId }, status: "succeeded" });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const wakeId = await seedDeferredWake({ companyId, agentId: foreignAgentId, issueId });
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
const candidate = await ports.writer.claimNextDeferredWake({ companyId, issueId: locked.primaryIssue.id });
expect(candidate?.id).toBe(wakeId);
const agent = await ports.reader.findInvokableAgent({ companyId, agentId: foreignAgentId });
expect(agent).toBeNull();
const failed = await ports.writer.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() });
expect(failed).toBe(true);
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
expect(result.outcome.kind).toBe("released");
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
expect(wakeRow?.status).toBe("failed");
expect(wakeRow?.error).toBe("Deferred wake could not be promoted: agent is not invokable");
expect(wakeRow?.runId).toBeNull();
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
expect(runs).toHaveLength(1);
expect(runs[0]!.id).toBe(runId);
});
// Review test (b): each release adapter mutation with a foreign company
// affects no row.
it("scopes every release mutation to its own company and affects no row across a company boundary", async () => {
const companyId = await seedCompany();
const otherCompanyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
await adapter.withIssueExecutionLock(
{ companyId, runId, now: new Date() },
async (_locked, ports) => {
const cancelledUnderWrongCompany = await ports.writer.cancelDeferredWake({
companyId: otherCompanyId,
wakeId,
reason: "cross-company cancel attempt",
now: new Date(),
});
expect(cancelledUnderWrongCompany).toBe(false);
const failedUnderWrongCompany = await ports.writer.failDeferredWake({
companyId: otherCompanyId,
wakeId,
now: new Date(),
});
expect(failedUnderWrongCompany).toBe(false);
const normalizedUnderWrongCompany = await ports.writer.normalizeDeferredWakeCommentIds({
companyId: otherCompanyId,
wakeId,
payload: { issueId },
liveCommentIds: ["c1"],
now: new Date(),
});
expect(normalizedUnderWrongCompany).toBeNull();
const reopenedUnderWrongCompany = await ports.writer.reopenIssue({
companyId: otherCompanyId,
issueId,
runId,
});
expect(reopenedUnderWrongCompany).toBeNull();
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
},
);
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
expect(wakeRow?.status).toBe("deferred_issue_execution");
expect(wakeRow?.error).toBeNull();
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
expect(issueRow?.status).toBe("blocked");
expect(issueRow?.executionState).toEqual({ phase: "running" });
});
// Review defect: the reopen path must carry the company into every read,
// lock, and write. A check before the write is not a boundary, because
// `issues.company_id` can change between that check and the write.
it("refuses to reopen an issue for a company that does not own it, and leaves the issue untouched", async () => {
const companyId = await seedCompany();
const otherCompanyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
reopened: null,
};
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
captured.reopened = await ports.writer.reopenIssue({ companyId: otherCompanyId, issueId, runId });
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
expect(captured.reopened).toBeNull();
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
expect(issueRow?.status).toBe("blocked");
expect(issueRow?.executionState).toEqual({ phase: "running" });
});
it("reopens an issue for the company that owns it, and clears the execution state", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
reopened: null,
};
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
captured.reopened = await ports.writer.reopenIssue({ companyId, issueId, runId });
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
expect(captured.reopened?.status).toBe("todo");
expect(captured.reopened?.executionState).toBeNull();
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
expect(issueRow?.status).toBe("todo");
expect(issueRow?.executionState).toBeNull();
});
// Review test (c): a deferred-status compare-and-set that affects no row
// claims nothing, and no other write in the promotion path ever runs.
it("fails the promotion claim when the deferred-status compare-and-set loses the race, before any other write", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
// A concurrent finalization already claimed this wake before the promotion claim runs.
await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, wakeId));
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
const claimed = await ports.writer.claimDeferredWakeForPromotion({ companyId, wakeId, now: new Date() });
expect(claimed).toBe(false);
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
expect(result.outcome.kind).toBe("released");
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
expect(runs).toHaveLength(1);
expect(runs[0]!.id).toBe(runId);
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
expect(issueRow?.executionRunId).toBeNull();
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
expect(wakeRow?.status).toBe("cancelled");
});
// `finalizePromotedWake`'s own writes guard against clobbering state a
// concurrent write already changed: it never takes the issue's execution
// lock away from a run that already holds it, and it never overwrites a
// `runId` a wake row already carries. Both guards only matter as defense
// in depth today (the release drain calls this at most once per
// transaction), so this drives the port directly to prove the SQL itself,
// independent of that call pattern.
it("guards finalizePromotedWake's own writes against clobbering an already-set execution lock or runId", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
const wakeIdA = await seedDeferredWake({ companyId, agentId, issueId });
const wakeIdB = await seedDeferredWake({ companyId, agentId, issueId });
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
const deferredAgent = { id: agentId, companyId, name: "CodexCoder", invokable: true };
const finalizedRunIds: string[] = [];
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
const finalize = async (wakeId: string) => {
const promoted = await ports.writer.finalizePromotedWake({
companyId,
wakeId,
deferredAgent,
issue: locked.primaryIssue,
finishingRun: locked.run,
contextSnapshot: { issueId },
reason: "issue_execution_promoted",
source: "automation",
triggerDetail: null,
payload: {},
responsibleUserId: "responsible-user",
sessionBefore: null,
now: new Date(),
});
finalizedRunIds.push(promoted.id);
};
// The issue's execution lock is free; this call takes it.
await finalize(wakeIdA);
// The issue's execution lock is already held by the first call's run,
// so this call's issue-lock write must no-op even though a run is
// still created.
await finalize(wakeIdB);
// Repeating the same wake must not overwrite its now-set `runId`.
await finalize(wakeIdA);
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
const [runA, runB, runC] = finalizedRunIds;
const wakeRowA = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdA)))[0];
const wakeRowB = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdB)))[0];
expect(wakeRowA?.runId).toBe(runA);
expect(wakeRowB?.runId).toBe(runB);
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
expect(issueRow?.executionRunId).toBe(runA);
expect(issueRow?.executionRunId).not.toBe(runB);
expect(issueRow?.executionRunId).not.toBe(runC);
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
// All three finalize calls each still insert their own run row.
expect(runs.map((run) => run.id).sort()).toEqual([runId, runA, runB, runC].sort());
});
it("locks the context issue and every sibling issue in id order, and two concurrent releases do not deadlock", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const issueA = await seedIssue({ companyId, assigneeAgentId: agentId });
const issueB = await seedIssue({ companyId, assigneeAgentId: agentId });
const runA = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueA } });
const runB = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueB } });
await db.update(issues).set({ executionRunId: runA, checkoutRunId: runB }).where(eq(issues.id, issueA));
await db.update(issues).set({ executionRunId: runB, checkoutRunId: runA }).where(eq(issues.id, issueB));
const adapterA = createPostgresWakeQueueAdapter(db, stubDeps);
const adapterB = createPostgresWakeQueueAdapter(db, stubDeps);
const releaseA = adapterA.withIssueExecutionLock({ companyId, runId: runA, now: new Date() }, async (_locked, _ports) => {
await new Promise((resolve) => setTimeout(resolve, 25));
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
const releaseB = adapterB.withIssueExecutionLock({ companyId, runId: runB, now: new Date() }, async (_locked, _ports) => {
await new Promise((resolve) => setTimeout(resolve, 25));
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
});
await expect(Promise.all([releaseA, releaseB])).resolves.toBeDefined();
const rows = await db.select().from(issues).where(eq(issues.companyId, companyId));
for (const row of rows) {
expect(row.executionRunId).toBeNull();
expect(row.checkoutRunId).toBeNull();
}
});
it("clears both lock columns on every sibling and keeps a transferred executionRunId", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent({ companyId });
const finishingRunId = await seedRun({ companyId, agentId, contextSnapshot: {} });
const retryRunId = await seedRun({ companyId, agentId, contextSnapshot: {}, status: "queued" });
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, executionRunId: retryRunId, checkoutRunId: finishingRunId });
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
const result = await adapter.withIssueExecutionLock({ companyId, runId: finishingRunId, now: new Date() }, async () => ({
outcome: { kind: "released" as const },
postCommitEffects: [],
}));
expect(result.outcome.kind).toBe("released");
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
// executionRunId already pointed at the retry, not the finishing run, so it must survive.
expect(issueRow?.executionRunId).toBe(retryRunId);
expect(issueRow?.checkoutRunId).toBeNull();
});
});
@@ -0,0 +1,817 @@
import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agentWakeupRequests,
agents,
heartbeatRuns,
issueComments,
issueRecoveryActions,
issueRelations,
issues,
nativeRunFinalizations,
} from "@paperclipai/db";
import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js";
import { evaluateAgentInvokability } from "../../../services/agent-invokability.js";
import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js";
import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js";
import { issueService } from "../../../services/issues.js";
import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js";
import { withRecoveryContext } from "../../../services/recovery/status-only-context.js";
import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js";
import {
buildConfigurationIncompleteRecoveryNoticeSeed,
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildImmediateExecutionPathRecoveryNoticeSeed,
buildWorkspaceValidationRecoveryNoticeSeed,
} from "../../../services/recovery/stranded-notice.js";
import {
queuedCommentIdsFromWakePayload,
withQueuedCommentIdsInWakePayload,
} from "../../../services/issue-queued-comment-queue.js";
import { extractWakeCommentIds } from "../../run-dispatch/index.js";
import { hasInteractionContinuationWakeContext } from "../domain/context.js";
import type {
DeferredWakeCandidate,
InvokableAgentSnapshot,
IssueLockWriter,
IssueSnapshot,
LockedIssueExecution,
ReleaseTransactionResult,
RunSnapshot,
WakeQueueReader,
WakeQueueWriter,
} from "../application/ports.js";
import type { RunSummary } from "../application/types.js";
import { WakeQueueApplicationError } from "../application/types.js";
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete";
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery";
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery";
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
type IssueRow = typeof issues.$inferSelect;
function parseObject(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
function normalizeAgentNameKey(value: string | null | undefined): string | null {
if (typeof value !== "string") return null;
const normalized = value.trim().toLowerCase();
return normalized.length > 0 ? normalized : null;
}
function isWorkspaceValidationFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
}
function isConfigurationIncompleteFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
}
function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete);
return {
id: row.id,
companyId: row.companyId,
agentId: row.agentId,
status: row.status,
runtimeMode: row.runtimeMode,
errorCode: row.errorCode,
responsibleUserId: row.responsibleUserId,
contextSnapshot: parseObject(row.contextSnapshot),
configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null,
};
}
function toIssueSnapshot(row: IssueRow): IssueSnapshot {
return {
id: row.id,
companyId: row.companyId,
identifier: row.identifier ?? "",
status: row.status,
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
hiddenAt: row.hiddenAt,
originKind: row.originKind,
monitorNextCheckAt: row.monitorNextCheckAt,
executionState: (row.executionState as Record<string, unknown> | null) ?? null,
responsibleUserId: row.responsibleUserId,
parentId: row.parentId,
originId: row.originId,
originRunId: row.originRunId,
};
}
function toRunSummary(row: HeartbeatRunRow): RunSummary {
return {
id: row.id,
companyId: row.companyId,
agentId: row.agentId,
invocationSource: row.invocationSource,
triggerDetail: row.triggerDetail,
wakeupRequestId: row.wakeupRequestId,
};
}
function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate {
const payload = parseObject(row.payload);
const queuedCommentIds = queuedCommentIdsFromWakePayload(payload);
const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason);
const queuedReason = wakeReason ?? readNonEmptyString(row.reason);
const queuedWakeIsCommentOnly =
!queuedReason ||
queuedReason === "issue_commented" ||
queuedReason === "issue_reopened_via_comment" ||
queuedReason === "issue_comment_mentioned";
const preservesIndependentContinuation =
hasInteractionContinuationWakeContext(deferredContextSeed) ||
deferredContextSeed.resumeIntent === true ||
!queuedWakeIsCommentOnly;
return {
id: row.id,
companyId: row.companyId,
agentId: row.agentId,
reason: row.reason,
source: row.source,
triggerDetail: row.triggerDetail,
requestedByActorType: row.requestedByActorType,
requestedByActorId: row.requestedByActorId,
payload,
queuedCommentIds,
preservesIndependentContinuation,
deferredContextSeed,
deferredCommentIds,
wakeReason,
};
}
export type WakeQueuePostgresAdapterDeps = {
resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
getRoutineEnv: WakeQueueReader["getRoutineEnv"];
resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
};
function buildReader(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueReader {
return {
async findInvokableAgent({ companyId, agentId }): Promise<InvokableAgentSnapshot | null> {
const agent = await tx
.select()
.from(agents)
.where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)))
.then((rows) => rows[0] ?? null);
if (!agent) return null;
const companyAgents = await tx
.select({ id: agents.id, companyId: agents.companyId, name: agents.name, reportsTo: agents.reportsTo, status: agents.status })
.from(agents)
.where(eq(agents.companyId, companyId));
const invokability = evaluateAgentInvokability(agent, companyAgents);
return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable };
},
resolveResponsibleUserId: deps.resolveResponsibleUserId,
getRoutineEnv: deps.getRoutineEnv,
resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
};
}
function buildWriter(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueWriter {
const treeControlSvc = issueTreeControlService(tx);
const issuesSvc = issueService(tx);
return {
async claimNextDeferredWake({ companyId, issueId }) {
const row = await tx
.select()
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
),
)
.orderBy(asc(agentWakeupRequests.requestedAt))
.limit(1)
.then((rows) => rows[0] ?? null);
return row ? toDeferredWakeCandidate(row) : null;
},
async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) {
const rows = await tx
.select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId })
.from(issueComments)
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queuedCommentIds)));
const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId;
const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => {
const row = rows.find((candidate) => candidate.id === commentId);
return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId));
});
const containedSelfAuthoredComment = rows.some(
(row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId,
);
return { liveNonSelfCommentIds, containedSelfAuthoredComment };
},
async cancelDeferredWake({ companyId, wakeId, reason, now }) {
const rows = await tx
.update(agentWakeupRequests)
.set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now })
.where(
and(
eq(agentWakeupRequests.id, wakeId),
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
),
)
.returning({ id: agentWakeupRequests.id });
return rows.length > 0;
},
async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) {
const rows = await tx
.update(agentWakeupRequests)
.set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now })
.where(
and(
eq(agentWakeupRequests.id, wakeId),
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
),
)
.returning();
const row = rows[0];
return row ? toDeferredWakeCandidate(row) : null;
},
async failDeferredWake({ companyId, wakeId, now }) {
const rows = await tx
.update(agentWakeupRequests)
.set({
status: "failed",
finishedAt: now,
error: "Deferred wake could not be promoted: agent is not invokable",
updatedAt: now,
})
.where(
and(
eq(agentWakeupRequests.id, wakeId),
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
),
)
.returning({ id: agentWakeupRequests.id });
return rows.length > 0;
},
async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) {
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId);
if (!activePauseHold) {
return {
activePauseHold: false,
treeHoldInteractionWake: false,
holdId: null,
rootIssueId: null,
mode: null,
reason: null,
releasePolicy: null,
};
}
const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, {
companyId,
issueId,
agentId: wakeAgentId,
contextSnapshot: deferredContextSeed,
requestedByActorType,
requestedByActorId,
});
return {
activePauseHold: true,
treeHoldInteractionWake,
holdId: activePauseHold.holdId,
rootIssueId: activePauseHold.rootIssueId,
mode: activePauseHold.mode,
reason: activePauseHold.reason,
releasePolicy: activePauseHold.releasePolicy,
};
},
async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) {
const rows = await tx
.select({ createdByRunId: issueComments.createdByRunId })
.from(issueComments)
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds)));
return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) };
},
async reopenIssue({ companyId, issueId }) {
const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx);
return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null;
},
async claimDeferredWakeForPromotion({ companyId, wakeId, now }) {
const claimed = await tx
.update(agentWakeupRequests)
.set({
status: "queued",
reason: "issue_execution_promoted",
claimedAt: null,
finishedAt: null,
error: null,
updatedAt: now,
})
.where(
and(
eq(agentWakeupRequests.id, wakeId),
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
),
)
.returning({ id: agentWakeupRequests.id });
return claimed.length > 0;
},
async finalizePromotedWake(input) {
const newRun = await tx
.insert(heartbeatRuns)
.values({
companyId: input.deferredAgent.companyId,
agentId: input.deferredAgent.id,
invocationSource: input.source,
triggerDetail: input.triggerDetail,
status: "queued",
wakeupRequestId: input.wakeId,
contextSnapshot: input.contextSnapshot,
responsibleUserId: input.responsibleUserId,
sessionIdBefore: input.sessionBefore,
continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt),
})
.returning()
.then((rows) => rows[0]);
// `claimDeferredWakeForPromotion` already moved this row off
// `deferred_issue_execution` inside this same transaction, so no
// concurrent claimer can still match that guard; this extra `runId is
// null` guard only protects against writing the link twice.
await tx
.update(agentWakeupRequests)
.set({ runId: newRun.id, updatedAt: input.now })
.where(
and(
eq(agentWakeupRequests.id, input.wakeId),
eq(agentWakeupRequests.companyId, input.companyId),
isNull(agentWakeupRequests.runId),
),
);
// Promoted mention wakes are issue-scoped, not issue ownership
// transfers. The lock-clearing step earlier in this transaction
// already set `executionRunId` to null for this issue, so the `is
// null` guard only protects against taking the lock twice.
await tx
.update(issues)
.set({
executionRunId: newRun.id,
executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name),
executionLockedAt: input.now,
updatedAt: input.now,
})
.where(
and(
eq(issues.id, input.issue.id),
eq(issues.companyId, input.companyId),
eq(issues.assigneeAgentId, input.deferredAgent.id),
isNull(issues.executionRunId),
),
);
return toRunSummary(newRun);
},
async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) {
const row = await tx
.select({ id: heartbeatRuns.id })
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]),
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
sql`${heartbeatRuns.id} <> ${excludeRunId}`,
agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
return row !== null;
},
async hasExplicitBlockerPath({ companyId, issueId }) {
const row = await tx
.select({ issueId: issueRelations.issueId })
.from(issueRelations)
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
.where(
and(
eq(issueRelations.companyId, companyId),
eq(issueRelations.relatedIssueId, issueId),
eq(issueRelations.type, "blocks"),
eq(issues.companyId, companyId),
notInArray(issues.status, ["done", "cancelled"]),
isNull(issues.hiddenAt),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
return row !== null;
},
async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) {
return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc);
},
async buildBlockedRecoveryNotice({ noticeKind, issueStatus, finishingRun }) {
if (noticeKind === "workspace_validation") {
return { notice: buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE };
}
if (noticeKind === "configuration_incomplete") {
return {
notice: buildConfigurationIncompleteRecoveryNoticeSeed(finishingRun.configurationIncompletePayload),
recoveryCause: CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE,
};
}
if (noticeKind === "execution_review_participant") {
return {
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
};
}
return { notice: buildImmediateExecutionPathRecoveryNoticeSeed({ status: issueStatus }), recoveryCause: null };
},
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
const executionState = parseIssueExecutionState(issue.executionState);
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
companyId,
agentId: recoveryAgent.id,
source: "automation",
triggerDetail: "system",
reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
payload: withRecoveryContext(
{
issueId: issue.id,
retryOfRunId: finishingRun.id,
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
currentStageId: executionState?.currentStageId ?? null,
currentStageType: executionState?.currentStageType ?? null,
},
"normal_model",
),
status: "queued",
requestedByActorType: "system",
requestedByActorId: null,
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
companyId,
agentId: recoveryAgent.id,
invocationSource: "automation",
triggerDetail: "system",
status: "queued",
wakeupRequestId: wakeupRequest.id,
contextSnapshot: withRecoveryContext(
{
issueId: issue.id,
taskId: issue.id,
wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
source: "issue.execution_review_recovery",
retryOfRunId: finishingRun.id,
currentStageId: executionState?.currentStageId ?? null,
currentStageType: executionState?.currentStageType ?? null,
reviewRecoveryInstruction:
"The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
},
"normal_model",
),
sessionIdBefore: sessionBefore,
retryOfRunId: finishingRun.id,
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
await tx
.update(agentWakeupRequests)
.set({ runId: queuedRun.id, updatedAt: now })
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
await tx
.update(issues)
.set({
executionRunId: queuedRun.id,
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
executionLockedAt: now,
updatedAt: now,
})
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
return toRunSummary(queuedRun);
},
async queueImmediateRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed";
const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery";
const recoveryContextSnapshot = withRecoveryContext(
{
issueId: issue.id,
taskId: issue.id,
wakeReason: recoveryReason,
retryReason,
source: recoverySource,
retryOfRunId: finishingRun.id,
},
"normal_model",
);
const routineEnvContext = await deps.getRoutineEnv({ companyId, issue });
const responsibleUserId = await deps.resolveResponsibleUserId({
companyId,
contextSnapshot: recoveryContextSnapshot,
issue,
routineEnvContext,
requestedByActorType: "system",
requestedByActorId: null,
source: "automation",
triggerDetail: "system",
existingRunResponsibleUserId: finishingRun.responsibleUserId,
});
if (!responsibleUserId) {
throw new WakeQueueApplicationError(
"responsible_user_unresolved",
"Unable to resolve responsible user for recovery heartbeat run",
{
runId: finishingRun.id,
agentId: recoveryAgent.id,
companyId,
issueId: issue.id,
wakeReason: recoveryReason,
},
);
}
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
companyId,
agentId: recoveryAgent.id,
source: "automation",
triggerDetail: "system",
reason: recoveryReason,
payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"),
status: "queued",
requestedByActorType: "system",
requestedByActorId: null,
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
companyId,
agentId: recoveryAgent.id,
invocationSource: "automation",
triggerDetail: "system",
status: "queued",
wakeupRequestId: wakeupRequest.id,
contextSnapshot: recoveryContextSnapshot,
responsibleUserId,
sessionIdBefore: sessionBefore,
retryOfRunId: finishingRun.id,
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
await tx
.update(agentWakeupRequests)
.set({ runId: queuedRun.id, updatedAt: now })
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
await tx
.update(issues)
.set({
executionRunId: queuedRun.id,
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
executionLockedAt: now,
updatedAt: now,
})
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
return toRunSummary(queuedRun);
},
};
}
async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise<boolean> {
const applies =
run.runtimeMode === "native" &&
["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
issue.assigneeAgentId === run.agentId &&
!["done", "cancelled"].includes(issue.status);
if (!applies) return false;
const existing = await tx
.select({ id: issueRecoveryActions.id })
.from(issueRecoveryActions)
.where(
and(
eq(issueRecoveryActions.companyId, issue.companyId),
eq(issueRecoveryActions.sourceIssueId, issue.id),
or(
inArray(issueRecoveryActions.status, ["active", "escalated"]),
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`,
),
),
)
.limit(1);
if (!existing.length) {
await tx
.update(nativeRunFinalizations)
.set({
phase: "terminal_failure",
leaseOwner: null,
leaseExpiresAt: null,
nextAttemptAt: null,
recoveryState: "blocked",
failureCode: "native_continuation_requires_reconciliation",
updatedAt: now,
})
.where(
and(
eq(nativeRunFinalizations.companyId, issue.companyId),
eq(nativeRunFinalizations.runId, run.id),
isNull(nativeRunFinalizations.resultId),
),
);
await issueRecoveryActionService(tx).upsertSourceScoped({
companyId: issue.companyId,
sourceIssueId: issue.id,
kind: "active_run_watchdog",
ownerType: "board",
returnOwnerAgentId: run.agentId,
cause: "native_continuation_requires_reconciliation",
fingerprint: `native-continuation:${run.id}`,
evidence: { runId: run.id, originalFailureCode: run.errorCode },
nextAction:
"Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.",
maxAttempts: 3,
wakePolicy: null,
supersedeOnIdentityChange: true,
});
}
return true;
}
export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter {
return {
async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
return db.transaction(async (rawTx) => {
const tx = rawTx as unknown as Db;
const run = await tx
.select()
.from(heartbeatRuns)
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
.then((rows) => rows[0] ?? null);
if (!run) {
throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`);
}
const runSnapshot = toRunSnapshot(run);
const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
// Lock the context issue (if any) and every issue that still references this
// run, in id order, so two concurrent finalizations can never deadlock on
// each other's row-lock acquisition order.
await tx.execute(
contextIssueId
? sql`
select id from issues
where company_id = ${input.companyId}
and (
id = ${contextIssueId}
or execution_run_id = ${run.id}
or checkout_run_id = ${run.id}
)
order by id
for update
`
: sql`
select id from issues
where company_id = ${input.companyId}
and (execution_run_id = ${run.id} or checkout_run_id = ${run.id})
order by id
for update
`,
);
const candidateIssues = await tx
.select()
.from(issues)
.where(
and(
eq(issues.companyId, input.companyId),
contextIssueId
? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id))
: or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)),
),
)
.orderBy(asc(issues.id));
// Two separate updates: a retry can move `executionRunId` to a new run
// while `checkoutRunId` still points at this one finishing.
await tx
.update(issues)
.set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now })
.where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id)));
await tx
.update(issues)
.set({ checkoutRunId: null, updatedAt: input.now })
.where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id)));
const issueRow =
(contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null;
if (!issueRow || (issueRow.executionRunId && issueRow.executionRunId !== run.id)) {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
if (
(isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) &&
(issueRow.status === "todo" || issueRow.status === "in_progress") &&
!issueRow.assigneeUserId &&
issueRow.assigneeAgentId === run.agentId
) {
const configurationIncomplete = isConfigurationIncompleteFailedRun(run);
const notice = configurationIncomplete
? buildConfigurationIncompleteRecoveryNoticeSeed(runSnapshot.configurationIncompletePayload)
: buildWorkspaceValidationRecoveryNoticeSeed();
return {
outcome: {
kind: "blocked",
issue: toIssueSnapshot(issueRow),
previousStatus: issueRow.status as "todo" | "in_progress",
notice,
recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE : WORKSPACE_VALIDATION_RECOVERY_CAUSE,
},
postCommitEffects: [],
run: runSnapshot,
};
}
if (legacyExecutionNeedsReconciliation(run)) {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
// An operator stop never promotes old queued work by itself. The next
// explicit wake adopts those messages atomically when it queues a run.
if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) {
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
}
const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot };
const result = await fn(locked, { reader: buildReader(tx, deps), writer: buildWriter(tx, deps) });
return { ...result, run: runSnapshot };
});
},
};
}