mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 09:50:48 +01:00
Move fresh issue run and receipt writes into wake-queue
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
6 files changed
+231
-90
No files matched your search
@@ -316,6 +316,8 @@ Keep recovery policy changes separate from retry scheduling and execution change
|
||||
Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`.
|
||||
It owns wake coalescing and batching, queued-run claims, daily heartbeat caps,
|
||||
concurrency and priority checks, timer admission, and native status wake intents.
|
||||
The wake-queue module creates fresh task runs and binds their receipts inside
|
||||
heartbeat's issue admission transaction.
|
||||
`createHeartbeatQueue` binds these operations without querying or starting work.
|
||||
The service supplies lifecycle effects, execution callbacks, worktree gates, and
|
||||
its process-wide execution and wakeup promise sets. Forwarding callbacks preserve
|
||||
|
||||
@@ -85,6 +85,7 @@ import type {
|
||||
WakeQueueHost,
|
||||
WakeQueueTransaction,
|
||||
} from "../application/ports.js";
|
||||
import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js";
|
||||
import type { RunSummary } from "../application/types.js";
|
||||
|
||||
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
|
||||
@@ -1535,3 +1536,54 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createFreshIssueRunWriter(
|
||||
scope: TransactionScope,
|
||||
companyId: string,
|
||||
): FreshIssueRunWriter<HeartbeatRunRow> {
|
||||
const tx = requireAdmissionTx(scope, companyId);
|
||||
return {
|
||||
async saveReceipt(input) {
|
||||
if (input.kind === "insert") {
|
||||
const [receipt] = await tx.insert(agentWakeupRequests).values({
|
||||
...input.seed, companyId, status: "queued",
|
||||
}).returning();
|
||||
return receipt;
|
||||
}
|
||||
const [receipt] = await tx.update(agentWakeupRequests).set({
|
||||
payload: input.payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(),
|
||||
}).where(and(
|
||||
eq(agentWakeupRequests.id, input.wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, input.agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
isNull(agentWakeupRequests.runId),
|
||||
)).returning();
|
||||
return receipt;
|
||||
},
|
||||
async insertQueuedRun(seed) {
|
||||
const [run] = await tx.insert(heartbeatRuns).values({
|
||||
...seed, companyId, status: "queued",
|
||||
}).returning();
|
||||
return run;
|
||||
},
|
||||
async bindReceipts({ wakeId, runId, failedChatRetryReceiptId, adopted }) {
|
||||
const inCompany = eq(agentWakeupRequests.companyId, companyId);
|
||||
const primaryReceipt = and(inCompany, eq(agentWakeupRequests.id, wakeId));
|
||||
await tx.update(agentWakeupRequests).set({ runId, updatedAt: new Date() })
|
||||
.where(primaryReceipt);
|
||||
if (failedChatRetryReceiptId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(),
|
||||
}).where(and(inCompany, eq(agentWakeupRequests.id, failedChatRetryReceiptId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
}
|
||||
if (adopted) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
status: "coalesced", runId, finishedAt: new Date(), updatedAt: new Date(),
|
||||
}).where(and(inCompany, inArray(agentWakeupRequests.id, adopted.wakeIds)));
|
||||
await tx.update(agentWakeupRequests).set({ payload: adopted.payload }).where(primaryReceipt);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { DurableWakeAdmissionReceipt } from "./ports.js";
|
||||
import { readNonEmptyString } from "../domain/values.js";
|
||||
|
||||
type WakeReceiptSeed = {
|
||||
agentId: string;
|
||||
source: string;
|
||||
triggerDetail: string | null;
|
||||
reason: string | null;
|
||||
payload: Record<string, unknown> | null;
|
||||
requestedByActorType: string | null;
|
||||
requestedByActorId: string | null;
|
||||
idempotencyKey: string | null;
|
||||
};
|
||||
|
||||
type QueuedRunSeed = {
|
||||
id?: string;
|
||||
agentId: string;
|
||||
scopeKind: "issue" | "company";
|
||||
issueId: string | null;
|
||||
invocationSource: string;
|
||||
triggerDetail: string | null;
|
||||
responsibleUserId: string;
|
||||
wakeupRequestId: string;
|
||||
retryOfRunId: string | null;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
sessionIdBefore: string | null;
|
||||
continuationAttempt: number;
|
||||
scheduledRetryAttempt?: number;
|
||||
scheduledRetryReason?: "transient_failure";
|
||||
};
|
||||
|
||||
type SaveWakeReceipt =
|
||||
| { kind: "insert"; seed: WakeReceiptSeed & Partial<DurableWakeAdmissionReceipt> }
|
||||
| { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] };
|
||||
|
||||
export type FreshIssueRunWriter<Run> = {
|
||||
saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>;
|
||||
insertQueuedRun(seed: QueuedRunSeed): Promise<Run>;
|
||||
bindReceipts(input: {
|
||||
wakeId: string;
|
||||
runId: string;
|
||||
failedChatRetryReceiptId?: string;
|
||||
adopted?: { wakeIds: string[]; payload: Record<string, unknown> };
|
||||
}): Promise<void>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs under heartbeat's existing issue lock. The caller validates admission,
|
||||
* selects saved comments and records specialized continuation delivery between
|
||||
* run creation and receipt binding.
|
||||
*/
|
||||
export function createFreshIssueRunAdmission<Run>(writer: FreshIssueRunWriter<Run>) {
|
||||
return {
|
||||
createReceipt(input: WakeReceiptSeed & {
|
||||
durableReceipt?: DurableWakeAdmissionReceipt;
|
||||
deferredChatReceiptId?: string;
|
||||
}) {
|
||||
const { durableReceipt, deferredChatReceiptId, ...seed } = input;
|
||||
// A deferred durable chat wake keeps its original identity and source.
|
||||
return writer.saveReceipt(deferredChatReceiptId
|
||||
? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload }
|
||||
: {
|
||||
kind: "insert",
|
||||
seed: durableReceipt
|
||||
? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt }
|
||||
: seed,
|
||||
});
|
||||
},
|
||||
createRun(input: Omit<QueuedRunSeed,
|
||||
"id" | "scopeKind" | "issueId" | "scheduledRetryAttempt" | "scheduledRetryReason"
|
||||
> & {
|
||||
explicitContinuationRunId: string | null;
|
||||
reconciledSourceRunId: string | null;
|
||||
reconciledRestoreRetryCount: number | null;
|
||||
issueId: unknown;
|
||||
}) {
|
||||
const {
|
||||
explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed
|
||||
} = input;
|
||||
const scopedIssueId = readNonEmptyString(issueId);
|
||||
return writer.insertQueuedRun({
|
||||
...seed,
|
||||
...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}),
|
||||
scopeKind: scopedIssueId ? "issue" : "company",
|
||||
issueId: scopedIssueId,
|
||||
retryOfRunId: reconciledSourceRunId || seed.retryOfRunId,
|
||||
sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore,
|
||||
...(reconciledRestoreRetryCount !== null ? {
|
||||
scheduledRetryAttempt: reconciledRestoreRetryCount,
|
||||
scheduledRetryReason: "transient_failure",
|
||||
} : {}),
|
||||
});
|
||||
},
|
||||
bindReceipts: writer.bindReceipts,
|
||||
};
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
createAdmissionTransactionScope as buildAdmissionTransactionScope,
|
||||
createFreshIssueRunWriter,
|
||||
createPostgresWakeQueueAdapter,
|
||||
createWakeAdmissionReader,
|
||||
createWakeAdmissionWriter,
|
||||
} from "./adapters/postgres.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js";
|
||||
import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js";
|
||||
import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js";
|
||||
import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js";
|
||||
import {
|
||||
createDiscardQueuedComment,
|
||||
@@ -110,6 +112,11 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) {
|
||||
|
||||
export type WakeQueue = ReturnType<typeof createWakeQueue>;
|
||||
|
||||
/** Uses the transaction that already owns the issue's admission lock. */
|
||||
export function bindFreshIssueRunAdmission(companyId: string, tx: Db) {
|
||||
return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Composes the three queued-comment queue mutations (edit, reorder,
|
||||
* discard): the Postgres adapter, which owns the one transaction each
|
||||
|
||||
@@ -11,7 +11,7 @@ import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../..
|
||||
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
|
||||
import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js";
|
||||
import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js";
|
||||
import { createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { TransactionScope } from "../../modules/wake-queue/application/ports.js";
|
||||
import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js";
|
||||
import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js";
|
||||
@@ -494,6 +494,40 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
return { company, agent, issue, wake, run };
|
||||
}
|
||||
|
||||
it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => {
|
||||
const local = await fixture("queued");
|
||||
const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning();
|
||||
const [foreignAgent] = await db.insert(agents).values({
|
||||
companyId: foreignCompany.id, name: "Other agent", adapterType: "process",
|
||||
}).returning();
|
||||
const [foreignWake] = await db.insert(agentWakeupRequests).values({
|
||||
companyId: foreignCompany.id, agentId: foreignAgent.id, source: "automation",
|
||||
status: "deferred_issue_execution", payload: { original: true },
|
||||
}).returning();
|
||||
const [localDeferredWake] = await db.insert(agentWakeupRequests).values({
|
||||
companyId: local.company.id, agentId: local.agent.id, source: "automation",
|
||||
status: "deferred_issue_execution",
|
||||
}).returning();
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db);
|
||||
expect(await admission.createReceipt({
|
||||
deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation",
|
||||
triggerDetail: null, reason: null, payload: null, requestedByActorType: null,
|
||||
requestedByActorId: null, idempotencyKey: null,
|
||||
})).toBeUndefined();
|
||||
await admission.bindReceipts({
|
||||
wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id,
|
||||
adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } },
|
||||
});
|
||||
});
|
||||
|
||||
const [preserved] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, foreignWake.id));
|
||||
expect(preserved).toMatchObject({ status: "deferred_issue_execution", runId: null, payload: { original: true } });
|
||||
const [adopted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, localDeferredWake.id));
|
||||
expect(adopted).toMatchObject({ status: "coalesced", runId: local.run.id });
|
||||
});
|
||||
|
||||
async function pendingPredecessorFixture() {
|
||||
const f = await fixture("queued");
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
|
||||
@@ -218,7 +218,7 @@ import type { budgetService } from "../budgets.js";
|
||||
import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js";
|
||||
import type { instanceSettingsService } from "../instance-settings.js";
|
||||
import type { createRunDispatch } from "../../modules/run-dispatch/index.js";
|
||||
import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { bindFreshIssueRunAdmission, hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { crashPoint } from "@paperclipai/shared/crash-points";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
@@ -5069,29 +5069,20 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}, issueActivityPublications);
|
||||
}
|
||||
|
||||
const wakeupValues = {
|
||||
...durableReceiptFields,
|
||||
companyId: agent.companyId,
|
||||
const freshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db);
|
||||
const wakeupRequest = await freshIssueRun.createReceipt({
|
||||
durableReceipt: receiptRequest,
|
||||
deferredChatReceiptId: durableRequest && executionWaitRequestId === durableRequest.id
|
||||
? durableRequest.id : undefined,
|
||||
agentId,
|
||||
source,
|
||||
triggerDetail,
|
||||
reason,
|
||||
payload,
|
||||
status: "queued" as const,
|
||||
requestedByActorType: opts.requestedByActorType ?? null,
|
||||
requestedByActorId: opts.requestedByActorId ?? null,
|
||||
idempotencyKey: opts.idempotencyKey ?? null,
|
||||
};
|
||||
// A deferred durable chat wake already owns this ID and source.
|
||||
// Promote that receipt in the same transaction as its successor.
|
||||
const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id
|
||||
? await tx.update(agentWakeupRequests).set({
|
||||
payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, durableRequest.id),
|
||||
eq(agentWakeupRequests.companyId, agent.companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId)))
|
||||
.returning().then(rows => rows[0])
|
||||
: await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]);
|
||||
});
|
||||
if (!wakeupRequest) return { kind: "deferred" as const };
|
||||
|
||||
// A handoff changes the executor, not the owner of saved user input.
|
||||
@@ -5166,41 +5157,29 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
agent.companyId, issueId, agentId, adoptedCommentIds);
|
||||
}
|
||||
await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined });
|
||||
const newRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
...(explicitContinuation ? { id: explicitContinuationRunId } : {}),
|
||||
companyId: agent.companyId,
|
||||
agentId,
|
||||
scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company",
|
||||
issueId: readNonEmptyString(enrichedContextSnapshot.issueId),
|
||||
invocationSource: source,
|
||||
triggerDetail,
|
||||
status: "queued",
|
||||
responsibleUserId: await resolveQueuedResponsibleUserId(),
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
retryOfRunId: failedChatRetry
|
||||
? durableRequest!.failedRunRetry!.failedRunId
|
||||
: opts.failedRunId ?? automaticParentRunId,
|
||||
contextSnapshot: adoptedComments.length
|
||||
? withQueuedCommentIdsInRunContext(
|
||||
enrichedContextSnapshot,
|
||||
adoptedCommentIds,
|
||||
adoptedComments.map((wake) => wake.id),
|
||||
)
|
||||
: enrichedContextSnapshot,
|
||||
sessionIdBefore: explicitContinuation ? null : sessionBefore,
|
||||
continuationAttempt,
|
||||
...(reconciledSourceRunId
|
||||
? { retryOfRunId: reconciledSourceRunId }
|
||||
: {}),
|
||||
...(reconciledRestoreRetryCount !== null ? {
|
||||
scheduledRetryAttempt: reconciledRestoreRetryCount,
|
||||
scheduledRetryReason: "transient_failure",
|
||||
} : {}),
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
const newRun = await freshIssueRun.createRun({
|
||||
explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null,
|
||||
agentId,
|
||||
issueId: enrichedContextSnapshot.issueId,
|
||||
invocationSource: source,
|
||||
triggerDetail,
|
||||
responsibleUserId: await resolveQueuedResponsibleUserId(),
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
retryOfRunId: failedChatRetry
|
||||
? durableRequest!.failedRunRetry!.failedRunId
|
||||
: opts.failedRunId ?? automaticParentRunId,
|
||||
contextSnapshot: adoptedComments.length
|
||||
? withQueuedCommentIdsInRunContext(
|
||||
enrichedContextSnapshot,
|
||||
adoptedCommentIds,
|
||||
adoptedComments.map((wake) => wake.id),
|
||||
)
|
||||
: enrichedContextSnapshot,
|
||||
sessionIdBefore: sessionBefore,
|
||||
continuationAttempt,
|
||||
reconciledSourceRunId,
|
||||
reconciledRestoreRetryCount,
|
||||
});
|
||||
|
||||
if (ownsReconstructionContinuation) {
|
||||
await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({
|
||||
@@ -5216,44 +5195,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id,
|
||||
runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId });
|
||||
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
runId: newRun.id,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
|
||||
|
||||
if (failedChatRetry && opts.taskRetryRequestId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, issue.companyId),
|
||||
eq(agentWakeupRequests.id, opts.taskRetryRequestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
}
|
||||
|
||||
if (adoptedComments.length) {
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "coalesced",
|
||||
runId: newRun.id,
|
||||
finishedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
inArray(
|
||||
agentWakeupRequests.id,
|
||||
adoptedComments.map((wake) => wake.id),
|
||||
),
|
||||
);
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds),
|
||||
})
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
|
||||
}
|
||||
await freshIssueRun.bindReceipts({
|
||||
wakeId: wakeupRequest.id,
|
||||
runId: newRun.id,
|
||||
failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined,
|
||||
adopted: adoptedComments.length ? {
|
||||
wakeIds: adoptedComments.map((wake) => wake.id),
|
||||
payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds),
|
||||
} : undefined,
|
||||
});
|
||||
|
||||
// executionRunId is NOT stamped here (enqueueWakeup queues the run but
|
||||
// doesn't start it). It will be stamped in claimQueuedRun() once the run
|
||||
|
||||
Reference in new issue
Block a user