diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index 055550e6773..ba0ce713ae4 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -316,6 +316,8 @@ Keep recovery policy changes separate from retry scheduling and execution change Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`. It owns wake coalescing and batching, queued-run claims, daily heartbeat caps, concurrency and priority checks, timer admission, and native status wake intents. +The wake-queue module creates fresh task runs and binds their receipts inside +heartbeat's issue admission transaction. `createHeartbeatQueue` binds these operations without querying or starting work. The service supplies lifecycle effects, execution callbacks, worktree gates, and its process-wide execution and wakeup promise sets. Forwarding callbacks preserve diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b3bbe973f9d..a70fa7b926a 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -85,6 +85,7 @@ import type { WakeQueueHost, WakeQueueTransaction, } from "../application/ports.js"; +import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js"; import type { RunSummary } from "../application/types.js"; const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; @@ -1535,3 +1536,54 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd }, }; } + +export function createFreshIssueRunWriter( + scope: TransactionScope, + companyId: string, +): FreshIssueRunWriter { + const tx = requireAdmissionTx(scope, companyId); + return { + async saveReceipt(input) { + if (input.kind === "insert") { + const [receipt] = await tx.insert(agentWakeupRequests).values({ + ...input.seed, companyId, status: "queued", + }).returning(); + return receipt; + } + const [receipt] = await tx.update(agentWakeupRequests).set({ + payload: input.payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), + }).where(and( + eq(agentWakeupRequests.id, input.wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, input.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + isNull(agentWakeupRequests.runId), + )).returning(); + return receipt; + }, + async insertQueuedRun(seed) { + const [run] = await tx.insert(heartbeatRuns).values({ + ...seed, companyId, status: "queued", + }).returning(); + return run; + }, + async bindReceipts({ wakeId, runId, failedChatRetryReceiptId, adopted }) { + const inCompany = eq(agentWakeupRequests.companyId, companyId); + const primaryReceipt = and(inCompany, eq(agentWakeupRequests.id, wakeId)); + await tx.update(agentWakeupRequests).set({ runId, updatedAt: new Date() }) + .where(primaryReceipt); + if (failedChatRetryReceiptId) { + await tx.update(agentWakeupRequests).set({ + status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(), + }).where(and(inCompany, eq(agentWakeupRequests.id, failedChatRetryReceiptId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + } + if (adopted) { + await tx.update(agentWakeupRequests).set({ + status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(), + }).where(and(inCompany, inArray(agentWakeupRequests.id, adopted.wakeIds))); + await tx.update(agentWakeupRequests).set({ payload: adopted.payload }).where(primaryReceipt); + } + }, + }; +} diff --git a/server/src/modules/wake-queue/application/fresh-issue-run.ts b/server/src/modules/wake-queue/application/fresh-issue-run.ts new file mode 100644 index 00000000000..52bebb8a082 --- /dev/null +++ b/server/src/modules/wake-queue/application/fresh-issue-run.ts @@ -0,0 +1,96 @@ +import type { DurableWakeAdmissionReceipt } from "./ports.js"; +import { readNonEmptyString } from "../domain/values.js"; + +type WakeReceiptSeed = { + agentId: string; + source: string; + triggerDetail: string | null; + reason: string | null; + payload: Record | null; + requestedByActorType: string | null; + requestedByActorId: string | null; + idempotencyKey: string | null; +}; + +type QueuedRunSeed = { + id?: string; + agentId: string; + scopeKind: "issue" | "company"; + issueId: string | null; + invocationSource: string; + triggerDetail: string | null; + responsibleUserId: string; + wakeupRequestId: string; + retryOfRunId: string | null; + contextSnapshot: Record; + sessionIdBefore: string | null; + continuationAttempt: number; + scheduledRetryAttempt?: number; + scheduledRetryReason?: "transient_failure"; +}; + +type SaveWakeReceipt = + | { kind: "insert"; seed: WakeReceiptSeed & Partial } + | { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] }; + +export type FreshIssueRunWriter = { + saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>; + insertQueuedRun(seed: QueuedRunSeed): Promise; + bindReceipts(input: { + wakeId: string; + runId: string; + failedChatRetryReceiptId?: string; + adopted?: { wakeIds: string[]; payload: Record }; + }): Promise; +}; + +/** + * Runs under heartbeat's existing issue lock. The caller validates admission, + * selects saved comments and records specialized continuation delivery between + * run creation and receipt binding. + */ +export function createFreshIssueRunAdmission(writer: FreshIssueRunWriter) { + return { + createReceipt(input: WakeReceiptSeed & { + durableReceipt?: DurableWakeAdmissionReceipt; + deferredChatReceiptId?: string; + }) { + const { durableReceipt, deferredChatReceiptId, ...seed } = input; + // A deferred durable chat wake keeps its original identity and source. + return writer.saveReceipt(deferredChatReceiptId + ? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload } + : { + kind: "insert", + seed: durableReceipt + ? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } + : seed, + }); + }, + createRun(input: Omit & { + explicitContinuationRunId: string | null; + reconciledSourceRunId: string | null; + reconciledRestoreRetryCount: number | null; + issueId: unknown; + }) { + const { + explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed + } = input; + const scopedIssueId = readNonEmptyString(issueId); + return writer.insertQueuedRun({ + ...seed, + ...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}), + scopeKind: scopedIssueId ? "issue" : "company", + issueId: scopedIssueId, + retryOfRunId: reconciledSourceRunId || seed.retryOfRunId, + sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore, + ...(reconciledRestoreRetryCount !== null ? { + scheduledRetryAttempt: reconciledRestoreRetryCount, + scheduledRetryReason: "transient_failure", + } : {}), + }); + }, + bindReceipts: writer.bindReceipts, + }; +} diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 9ac0a76879b..33fd109e102 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -1,12 +1,14 @@ import type { Db } from "@paperclipai/db"; import { createAdmissionTransactionScope as buildAdmissionTransactionScope, + createFreshIssueRunWriter, createPostgresWakeQueueAdapter, createWakeAdmissionReader, createWakeAdmissionWriter, } from "./adapters/postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js"; +import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js"; import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js"; import { createDiscardQueuedComment, @@ -110,6 +112,11 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) { export type WakeQueue = ReturnType; +/** Uses the transaction that already owns the issue's admission lock. */ +export function bindFreshIssueRunAdmission(companyId: string, tx: Db) { + return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId)); +} + /** * Composes the three queued-comment queue mutations (edit, reorder, * discard): the Postgres adapter, which owns the one transaction each diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 33c62c65645..a67dd4cc4a9 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -11,7 +11,7 @@ import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../.. import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; -import { createWakeQueue } from "../../modules/wake-queue/index.js"; +import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js"; import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; @@ -494,6 +494,40 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { company, agent, issue, wake, run }; } + it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => { + const local = await fixture("queued"); + const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); + const [foreignAgent] = await db.insert(agents).values({ + companyId: foreignCompany.id, name: "Other agent", adapterType: "process", + }).returning(); + const [foreignWake] = await db.insert(agentWakeupRequests).values({ + companyId: foreignCompany.id, agentId: foreignAgent.id, source: "automation", + status: "deferred_issue_execution", payload: { original: true }, + }).returning(); + const [localDeferredWake] = await db.insert(agentWakeupRequests).values({ + companyId: local.company.id, agentId: local.agent.id, source: "automation", + status: "deferred_issue_execution", + }).returning(); + + await db.transaction(async (tx) => { + const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db); + expect(await admission.createReceipt({ + deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation", + triggerDetail: null, reason: null, payload: null, requestedByActorType: null, + requestedByActorId: null, idempotencyKey: null, + })).toBeUndefined(); + await admission.bindReceipts({ + wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id, + adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } }, + }); + }); + + const [preserved] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, foreignWake.id)); + expect(preserved).toMatchObject({ status: "deferred_issue_execution", runId: null, payload: { original: true } }); + const [adopted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, localDeferredWake.id)); + expect(adopted).toMatchObject({ status: "coalesced", runId: local.run.id }); + }); + async function pendingPredecessorFixture() { const f = await fixture("queued"); const [source] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 8d575a55ef9..79da5a3f8c7 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -218,7 +218,7 @@ import type { budgetService } from "../budgets.js"; import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js"; import type { instanceSettingsService } from "../instance-settings.js"; import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; -import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; +import { bindFreshIssueRunAdmission, hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; import { crashPoint } from "@paperclipai/shared/crash-points"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -5069,29 +5069,20 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }, issueActivityPublications); } - const wakeupValues = { - ...durableReceiptFields, - companyId: agent.companyId, + const freshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db); + const wakeupRequest = await freshIssueRun.createReceipt({ + durableReceipt: receiptRequest, + deferredChatReceiptId: durableRequest && executionWaitRequestId === durableRequest.id + ? durableRequest.id : undefined, agentId, source, triggerDetail, reason, payload, - status: "queued" as const, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, - }; - // A deferred durable chat wake already owns this ID and source. - // Promote that receipt in the same transaction as its successor. - const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id - ? await tx.update(agentWakeupRequests).set({ - payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, durableRequest.id), - eq(agentWakeupRequests.companyId, agent.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId))) - .returning().then(rows => rows[0]) - : await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]); + }); if (!wakeupRequest) return { kind: "deferred" as const }; // A handoff changes the executor, not the owner of saved user input. @@ -5166,41 +5157,29 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende agent.companyId, issueId, agentId, adoptedCommentIds); } await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); - const newRun = await tx - .insert(heartbeatRuns) - .values({ - ...(explicitContinuation ? { id: explicitContinuationRunId } : {}), - companyId: agent.companyId, - agentId, - scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", - issueId: readNonEmptyString(enrichedContextSnapshot.issueId), - invocationSource: source, - triggerDetail, - status: "queued", - responsibleUserId: await resolveQueuedResponsibleUserId(), - wakeupRequestId: wakeupRequest.id, - retryOfRunId: failedChatRetry - ? durableRequest!.failedRunRetry!.failedRunId - : opts.failedRunId ?? automaticParentRunId, - contextSnapshot: adoptedComments.length - ? withQueuedCommentIdsInRunContext( - enrichedContextSnapshot, - adoptedCommentIds, - adoptedComments.map((wake) => wake.id), - ) - : enrichedContextSnapshot, - sessionIdBefore: explicitContinuation ? null : sessionBefore, - continuationAttempt, - ...(reconciledSourceRunId - ? { retryOfRunId: reconciledSourceRunId } - : {}), - ...(reconciledRestoreRetryCount !== null ? { - scheduledRetryAttempt: reconciledRestoreRetryCount, - scheduledRetryReason: "transient_failure", - } : {}), - }) - .returning() - .then((rows) => rows[0]); + const newRun = await freshIssueRun.createRun({ + explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null, + agentId, + issueId: enrichedContextSnapshot.issueId, + invocationSource: source, + triggerDetail, + responsibleUserId: await resolveQueuedResponsibleUserId(), + wakeupRequestId: wakeupRequest.id, + retryOfRunId: failedChatRetry + ? durableRequest!.failedRunRetry!.failedRunId + : opts.failedRunId ?? automaticParentRunId, + contextSnapshot: adoptedComments.length + ? withQueuedCommentIdsInRunContext( + enrichedContextSnapshot, + adoptedCommentIds, + adoptedComments.map((wake) => wake.id), + ) + : enrichedContextSnapshot, + sessionIdBefore: sessionBefore, + continuationAttempt, + reconciledSourceRunId, + reconciledRestoreRetryCount, + }); if (ownsReconstructionContinuation) { await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ @@ -5216,44 +5195,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); - await tx - .update(agentWakeupRequests) - .set({ - runId: newRun.id, - updatedAt: new Date(), - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - if (failedChatRetry && opts.taskRetryRequestId) { - await tx.update(agentWakeupRequests).set({ - status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.id, opts.taskRetryRequestId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); - } - - if (adoptedComments.length) { - await tx - .update(agentWakeupRequests) - .set({ - status: "coalesced", - runId: newRun.id, - finishedAt: new Date(), - updatedAt: new Date(), - }) - .where( - inArray( - agentWakeupRequests.id, - adoptedComments.map((wake) => wake.id), - ), - ); - await tx - .update(agentWakeupRequests) - .set({ - payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - } + await freshIssueRun.bindReceipts({ + wakeId: wakeupRequest.id, + runId: newRun.id, + failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined, + adopted: adoptedComments.length ? { + wakeIds: adoptedComments.map((wake) => wake.id), + payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), + } : undefined, + }); // executionRunId is NOT stamped here (enqueueWakeup queues the run but // doesn't start it). It will be stamped in claimQueuedRun() once the run