Move fresh issue run and receipt writes into wake-queue

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip committed 2026-11-18 16:53:29 -08:00
1 parent 92dbf39e6f
commit 7479acbd22
6 files changed
+231 -90

No files matched your search

+2
View File
@@ -316,6 +316,8 @@ Keep recovery policy changes separate from retry scheduling and execution change
Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`. Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`.
It owns wake coalescing and batching, queued-run claims, daily heartbeat caps, It owns wake coalescing and batching, queued-run claims, daily heartbeat caps,
concurrency and priority checks, timer admission, and native status wake intents. concurrency and priority checks, timer admission, and native status wake intents.
The wake-queue module creates fresh task runs and binds their receipts inside
heartbeat's issue admission transaction.
`createHeartbeatQueue` binds these operations without querying or starting work. `createHeartbeatQueue` binds these operations without querying or starting work.
The service supplies lifecycle effects, execution callbacks, worktree gates, and The service supplies lifecycle effects, execution callbacks, worktree gates, and
its process-wide execution and wakeup promise sets. Forwarding callbacks preserve its process-wide execution and wakeup promise sets. Forwarding callbacks preserve
@@ -85,6 +85,7 @@ import type {
WakeQueueHost, WakeQueueHost,
WakeQueueTransaction, WakeQueueTransaction,
} from "../application/ports.js"; } from "../application/ports.js";
import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js";
import type { RunSummary } from "../application/types.js"; import type { RunSummary } from "../application/types.js";
const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
@@ -1535,3 +1536,54 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
}, },
}; };
} }
export function createFreshIssueRunWriter(
scope: TransactionScope,
companyId: string,
): FreshIssueRunWriter<HeartbeatRunRow> {
const tx = requireAdmissionTx(scope, companyId);
return {
async saveReceipt(input) {
if (input.kind === "insert") {
const [receipt] = await tx.insert(agentWakeupRequests).values({
...input.seed, companyId, status: "queued",
}).returning();
return receipt;
}
const [receipt] = await tx.update(agentWakeupRequests).set({
payload: input.payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(),
}).where(and(
eq(agentWakeupRequests.id, input.wakeId),
eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.agentId, input.agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution"),
isNull(agentWakeupRequests.runId),
)).returning();
return receipt;
},
async insertQueuedRun(seed) {
const [run] = await tx.insert(heartbeatRuns).values({
...seed, companyId, status: "queued",
}).returning();
return run;
},
async bindReceipts({ wakeId, runId, failedChatRetryReceiptId, adopted }) {
const inCompany = eq(agentWakeupRequests.companyId, companyId);
const primaryReceipt = and(inCompany, eq(agentWakeupRequests.id, wakeId));
await tx.update(agentWakeupRequests).set({ runId, updatedAt: new Date() })
.where(primaryReceipt);
if (failedChatRetryReceiptId) {
await tx.update(agentWakeupRequests).set({
status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(),
}).where(and(inCompany, eq(agentWakeupRequests.id, failedChatRetryReceiptId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
}
if (adopted) {
await tx.update(agentWakeupRequests).set({
status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(),
}).where(and(inCompany, inArray(agentWakeupRequests.id, adopted.wakeIds)));
await tx.update(agentWakeupRequests).set({ payload: adopted.payload }).where(primaryReceipt);
}
},
};
}
@@ -0,0 +1,96 @@
import type { DurableWakeAdmissionReceipt } from "./ports.js";
import { readNonEmptyString } from "../domain/values.js";
type WakeReceiptSeed = {
agentId: string;
source: string;
triggerDetail: string | null;
reason: string | null;
payload: Record<string, unknown> | null;
requestedByActorType: string | null;
requestedByActorId: string | null;
idempotencyKey: string | null;
};
type QueuedRunSeed = {
id?: string;
agentId: string;
scopeKind: "issue" | "company";
issueId: string | null;
invocationSource: string;
triggerDetail: string | null;
responsibleUserId: string;
wakeupRequestId: string;
retryOfRunId: string | null;
contextSnapshot: Record<string, unknown>;
sessionIdBefore: string | null;
continuationAttempt: number;
scheduledRetryAttempt?: number;
scheduledRetryReason?: "transient_failure";
};
type SaveWakeReceipt =
| { kind: "insert"; seed: WakeReceiptSeed & Partial<DurableWakeAdmissionReceipt> }
| { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] };
export type FreshIssueRunWriter<Run> = {
saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>;
insertQueuedRun(seed: QueuedRunSeed): Promise<Run>;
bindReceipts(input: {
wakeId: string;
runId: string;
failedChatRetryReceiptId?: string;
adopted?: { wakeIds: string[]; payload: Record<string, unknown> };
}): Promise<void>;
};
/**
* Runs under heartbeat's existing issue lock. The caller validates admission,
* selects saved comments and records specialized continuation delivery between
* run creation and receipt binding.
*/
export function createFreshIssueRunAdmission<Run>(writer: FreshIssueRunWriter<Run>) {
return {
createReceipt(input: WakeReceiptSeed & {
durableReceipt?: DurableWakeAdmissionReceipt;
deferredChatReceiptId?: string;
}) {
const { durableReceipt, deferredChatReceiptId, ...seed } = input;
// A deferred durable chat wake keeps its original identity and source.
return writer.saveReceipt(deferredChatReceiptId
? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload }
: {
kind: "insert",
seed: durableReceipt
? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt }
: seed,
});
},
createRun(input: Omit<QueuedRunSeed,
"id" | "scopeKind" | "issueId" | "scheduledRetryAttempt" | "scheduledRetryReason"
> & {
explicitContinuationRunId: string | null;
reconciledSourceRunId: string | null;
reconciledRestoreRetryCount: number | null;
issueId: unknown;
}) {
const {
explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed
} = input;
const scopedIssueId = readNonEmptyString(issueId);
return writer.insertQueuedRun({
...seed,
...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}),
scopeKind: scopedIssueId ? "issue" : "company",
issueId: scopedIssueId,
retryOfRunId: reconciledSourceRunId || seed.retryOfRunId,
sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore,
...(reconciledRestoreRetryCount !== null ? {
scheduledRetryAttempt: reconciledRestoreRetryCount,
scheduledRetryReason: "transient_failure",
} : {}),
});
},
bindReceipts: writer.bindReceipts,
};
}
+7
View File
@@ -1,12 +1,14 @@
import type { Db } from "@paperclipai/db"; import type { Db } from "@paperclipai/db";
import { import {
createAdmissionTransactionScope as buildAdmissionTransactionScope, createAdmissionTransactionScope as buildAdmissionTransactionScope,
createFreshIssueRunWriter,
createPostgresWakeQueueAdapter, createPostgresWakeQueueAdapter,
createWakeAdmissionReader, createWakeAdmissionReader,
createWakeAdmissionWriter, createWakeAdmissionWriter,
} from "./adapters/postgres.js"; } from "./adapters/postgres.js";
import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js";
import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js";
import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js";
import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js"; import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js";
import { import {
createDiscardQueuedComment, createDiscardQueuedComment,
@@ -110,6 +112,11 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) {
export type WakeQueue = ReturnType<typeof createWakeQueue>; export type WakeQueue = ReturnType<typeof createWakeQueue>;
/** Uses the transaction that already owns the issue's admission lock. */
export function bindFreshIssueRunAdmission(companyId: string, tx: Db) {
return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId));
}
/** /**
* Composes the three queued-comment queue mutations (edit, reorder, * Composes the three queued-comment queue mutations (edit, reorder,
* discard): the Postgres adapter, which owns the one transaction each * discard): the Postgres adapter, which owns the one transaction each
+35 -1
View File
@@ -11,7 +11,7 @@ import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../..
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js";
import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js";
import { createWakeQueue } from "../../modules/wake-queue/index.js"; import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js";
import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { TransactionScope } from "../../modules/wake-queue/application/ports.js";
import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js";
import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js";
@@ -494,6 +494,40 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
return { company, agent, issue, wake, run }; return { company, agent, issue, wake, run };
} }
it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => {
const local = await fixture("queued");
const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning();
const [foreignAgent] = await db.insert(agents).values({
companyId: foreignCompany.id, name: "Other agent", adapterType: "process",
}).returning();
const [foreignWake] = await db.insert(agentWakeupRequests).values({
companyId: foreignCompany.id, agentId: foreignAgent.id, source: "automation",
status: "deferred_issue_execution", payload: { original: true },
}).returning();
const [localDeferredWake] = await db.insert(agentWakeupRequests).values({
companyId: local.company.id, agentId: local.agent.id, source: "automation",
status: "deferred_issue_execution",
}).returning();
await db.transaction(async (tx) => {
const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db);
expect(await admission.createReceipt({
deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation",
triggerDetail: null, reason: null, payload: null, requestedByActorType: null,
requestedByActorId: null, idempotencyKey: null,
})).toBeUndefined();
await admission.bindReceipts({
wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id,
adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } },
});
});
const [preserved] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, foreignWake.id));
expect(preserved).toMatchObject({ status: "deferred_issue_execution", runId: null, payload: { original: true } });
const [adopted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, localDeferredWake.id));
expect(adopted).toMatchObject({ status: "coalesced", runId: local.run.id });
});
async function pendingPredecessorFixture() { async function pendingPredecessorFixture() {
const f = await fixture("queued"); const f = await fixture("queued");
const [source] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, const [source] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
+39 -89
View File
@@ -218,7 +218,7 @@ import type { budgetService } from "../budgets.js";
import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js"; import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js";
import type { instanceSettingsService } from "../instance-settings.js"; import type { instanceSettingsService } from "../instance-settings.js";
import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; import type { createRunDispatch } from "../../modules/run-dispatch/index.js";
import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; import { bindFreshIssueRunAdmission, hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
import { crashPoint } from "@paperclipai/shared/crash-points"; import { crashPoint } from "@paperclipai/shared/crash-points";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
@@ -5069,29 +5069,20 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}, issueActivityPublications); }, issueActivityPublications);
} }
const wakeupValues = { const freshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db);
...durableReceiptFields, const wakeupRequest = await freshIssueRun.createReceipt({
companyId: agent.companyId, durableReceipt: receiptRequest,
deferredChatReceiptId: durableRequest && executionWaitRequestId === durableRequest.id
? durableRequest.id : undefined,
agentId, agentId,
source, source,
triggerDetail, triggerDetail,
reason, reason,
payload, payload,
status: "queued" as const,
requestedByActorType: opts.requestedByActorType ?? null, requestedByActorType: opts.requestedByActorType ?? null,
requestedByActorId: opts.requestedByActorId ?? null, requestedByActorId: opts.requestedByActorId ?? null,
idempotencyKey: opts.idempotencyKey ?? null, idempotencyKey: opts.idempotencyKey ?? null,
}; });
// A deferred durable chat wake already owns this ID and source.
// Promote that receipt in the same transaction as its successor.
const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id
? await tx.update(agentWakeupRequests).set({
payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, durableRequest.id),
eq(agentWakeupRequests.companyId, agent.companyId),
eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId)))
.returning().then(rows => rows[0])
: await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]);
if (!wakeupRequest) return { kind: "deferred" as const }; if (!wakeupRequest) return { kind: "deferred" as const };
// A handoff changes the executor, not the owner of saved user input. // A handoff changes the executor, not the owner of saved user input.
@@ -5166,41 +5157,29 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
agent.companyId, issueId, agentId, adoptedCommentIds); agent.companyId, issueId, agentId, adoptedCommentIds);
} }
await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined });
const newRun = await tx const newRun = await freshIssueRun.createRun({
.insert(heartbeatRuns) explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null,
.values({ agentId,
...(explicitContinuation ? { id: explicitContinuationRunId } : {}), issueId: enrichedContextSnapshot.issueId,
companyId: agent.companyId, invocationSource: source,
agentId, triggerDetail,
scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", responsibleUserId: await resolveQueuedResponsibleUserId(),
issueId: readNonEmptyString(enrichedContextSnapshot.issueId), wakeupRequestId: wakeupRequest.id,
invocationSource: source, retryOfRunId: failedChatRetry
triggerDetail, ? durableRequest!.failedRunRetry!.failedRunId
status: "queued", : opts.failedRunId ?? automaticParentRunId,
responsibleUserId: await resolveQueuedResponsibleUserId(), contextSnapshot: adoptedComments.length
wakeupRequestId: wakeupRequest.id, ? withQueuedCommentIdsInRunContext(
retryOfRunId: failedChatRetry enrichedContextSnapshot,
? durableRequest!.failedRunRetry!.failedRunId adoptedCommentIds,
: opts.failedRunId ?? automaticParentRunId, adoptedComments.map((wake) => wake.id),
contextSnapshot: adoptedComments.length )
? withQueuedCommentIdsInRunContext( : enrichedContextSnapshot,
enrichedContextSnapshot, sessionIdBefore: sessionBefore,
adoptedCommentIds, continuationAttempt,
adoptedComments.map((wake) => wake.id), reconciledSourceRunId,
) reconciledRestoreRetryCount,
: enrichedContextSnapshot, });
sessionIdBefore: explicitContinuation ? null : sessionBefore,
continuationAttempt,
...(reconciledSourceRunId
? { retryOfRunId: reconciledSourceRunId }
: {}),
...(reconciledRestoreRetryCount !== null ? {
scheduledRetryAttempt: reconciledRestoreRetryCount,
scheduledRetryReason: "transient_failure",
} : {}),
})
.returning()
.then((rows) => rows[0]);
if (ownsReconstructionContinuation) { if (ownsReconstructionContinuation) {
await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({
@@ -5216,44 +5195,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id,
runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId });
await tx await freshIssueRun.bindReceipts({
.update(agentWakeupRequests) wakeId: wakeupRequest.id,
.set({ runId: newRun.id,
runId: newRun.id, failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined,
updatedAt: new Date(), adopted: adoptedComments.length ? {
}) wakeIds: adoptedComments.map((wake) => wake.id),
.where(eq(agentWakeupRequests.id, wakeupRequest.id)); payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds),
} : undefined,
if (failedChatRetry && opts.taskRetryRequestId) { });
await tx.update(agentWakeupRequests).set({
status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, issue.companyId),
eq(agentWakeupRequests.id, opts.taskRetryRequestId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
}
if (adoptedComments.length) {
await tx
.update(agentWakeupRequests)
.set({
status: "coalesced",
runId: newRun.id,
finishedAt: new Date(),
updatedAt: new Date(),
})
.where(
inArray(
agentWakeupRequests.id,
adoptedComments.map((wake) => wake.id),
),
);
await tx
.update(agentWakeupRequests)
.set({
payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds),
})
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
}
// executionRunId is NOT stamped here (enqueueWakeup queues the run but // executionRunId is NOT stamped here (enqueueWakeup queues the run but
// doesn't start it). It will be stamped in claimQueuedRun() once the run // doesn't start it). It will be stamped in claimQueuedRun() once the run