Admit saved task comments with the fresh issue run

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip committed 2026-11-18 17:02:52 -08:00
1 parent 7479acbd22
commit c957f8529c
12 files changed
+372 -283

No files matched your search

+4 -2
View File
@@ -316,8 +316,10 @@ Keep recovery policy changes separate from retry scheduling and execution change
Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`.
It owns wake coalescing and batching, queued-run claims, daily heartbeat caps,
concurrency and priority checks, timer admission, and native status wake intents.
The wake-queue module creates fresh task runs and binds their receipts inside
heartbeat's issue admission transaction.
The wake-queue module admits fresh task runs inside heartbeat's issue admission
transaction. It selects eligible saved comment receipts in request order and
binds them to the new run. Heartbeat supplies responsible-user resolution and
records any specialized continuation delivery before those receipts are bound.
`createHeartbeatQueue` binds these operations without querying or starting work.
The service supplies lifecycle effects, execution callbacks, worktree gates, and
its process-wide execution and wakeup promise sets. Forwarding callbacks preserve
+88
View File
@@ -0,0 +1,88 @@
export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds";
function parseObject(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function uniqueIds(value: unknown): string[] {
if (!Array.isArray(value)) return [];
const seen = new Set<string>();
return value.flatMap((candidate) => {
if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return [];
seen.add(candidate);
return [candidate];
});
}
export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] {
const payload = parseObject(payloadValue);
const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
return uniqueIds(context[WAKE_COMMENT_IDS_KEY]);
}
export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] {
return uniqueIds(parseObject(contextValue)[WAKE_COMMENT_IDS_KEY]);
}
export function withQueuedCommentIdsInWakePayload(
payloadValue: unknown,
ids: string[],
options: { preserveSource?: boolean } = {},
): Record<string, unknown> {
const payload = { ...parseObject(payloadValue) };
const context = { ...parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]) };
if (ids.length > 0) {
const latestId = ids[ids.length - 1]!;
context[WAKE_COMMENT_IDS_KEY] = ids;
if (!options.preserveSource) {
context.wakeCommentId = latestId;
context.commentId = latestId;
payload.commentId = latestId;
}
} else {
delete context[WAKE_COMMENT_IDS_KEY];
delete context.wakeCommentId;
delete context.commentId;
delete payload.commentId;
}
payload[DEFERRED_WAKE_CONTEXT_KEY] = context;
return payload;
}
export function withQueuedCommentIdsInRunContext(
contextValue: unknown,
ids: string[],
sourceWakeIds?: string[],
): Record<string, unknown> {
const context = { ...parseObject(contextValue) };
if (sourceWakeIds) {
context.queuedCommentSourceWakeIds = uniqueIds([
...uniqueIds(context.queuedCommentSourceWakeIds),
...sourceWakeIds,
]);
}
if (ids.length > 0) {
const latestId = ids[ids.length - 1]!;
context[WAKE_COMMENT_IDS_KEY] = ids;
context.wakeCommentId = latestId;
context.commentId = latestId;
} else {
delete context[WAKE_COMMENT_IDS_KEY];
delete context.wakeCommentId;
delete context.commentId;
}
// These projections are generated immediately before dispatch. Any queue
// mutation must force them to be rebuilt from the canonical comment ids.
delete context.paperclipWake;
delete context.paperclipWakeComment;
delete context.paperclipTaskMarkdown;
delete context.paperclipTaskMarkdownCompact;
delete context.paperclipTaskMarkdownAssignment;
delete context.paperclipTaskMarkdownAssignmentCompact;
delete context.paperclipTurnContext;
return context;
}
@@ -5,7 +5,7 @@ import { readNativeWorkspaceSyncReference } from "../../../services/native-runti
import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
import { isAgentDirectoryPreparationWait } from "../../../services/pre-dispatch-execution-evidence.js";
import { isUserCommentContinuationReason } from "../../../services/explicit-native-continuation.js";
import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js";
import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../lib/queued-comment-context.js";
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "../../../services/cancelled-workspace-preservation-notifications.js";
@@ -5,6 +5,9 @@
// independently maintained copy in either file could silently drift out of
// sync with the other and change only one of the two gates that read it.
import { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js";
export { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js";
function parseObject(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
@@ -30,7 +33,6 @@ export function isPreparationWaitRetryReason(reason: string | null | undefined):
}
export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry";
export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry";
export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds";
export const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set([
"accepted",
"answered",
@@ -16,7 +16,7 @@ import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction-
import { currentConversationCommentCondition } from "../../../services/agent-conversations.js";
import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js";
import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js";
import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
import { and, asc, eq, inArray, isNull, ne, notInArray, or, sql } from "drizzle-orm";
import { readReleaseRun, type ReleaseRun } from "./release-run-read.js";
import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js";
import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js";
@@ -53,13 +53,15 @@ import { executionRetryAccounting } from "../../../services/execution-recovery-a
import { issueService } from "../../../services/issues.js";
import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js";
import { undeliveredLegacyUserCommentIds } from "../../../services/explicit-native-continuation.js";
import { withRecoveryContext } from "../../../services/recovery/status-only-context.js";
import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js";
import {
queuedCommentIdsFromWakePayload,
DEFERRED_WAKE_CONTEXT_KEY,
queuedCommentIdsFromRunContext,
queuedCommentIdsFromWakePayload,
withQueuedCommentIdsInWakePayload,
} from "../../../services/issue-queued-comment-queue.js";
} from "../../../lib/queued-comment-context.js";
import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds, readWakeChatConversationId } from "../../run-dispatch/index.js";
import { deferredWakeHoldsNoOtherConversation, otherConversationCommentIds, splitOtherConversationMessages } from "./chat-conversation-batch.js";
import { hasInteractionContinuationWakeContext } from "../domain/context.js";
@@ -85,13 +87,12 @@ import type {
WakeQueueHost,
WakeQueueTransaction,
} from "../application/ports.js";
import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js";
import type { FreshIssueRunStore } from "../application/fresh-issue-run.js";
import type { RunSummary } from "../application/types.js";
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
/** Wake outcome for a deferred chat wake whose news an earlier reply already gave. */
export const ALREADY_REPORTED_WAKE_REASON = "Already reported by an earlier reply in this conversation";
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
@@ -1537,10 +1538,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
};
}
export function createFreshIssueRunWriter(
export function createFreshIssueRunStore(
scope: TransactionScope,
companyId: string,
): FreshIssueRunWriter<HeartbeatRunRow> {
): FreshIssueRunStore<HeartbeatRunRow> {
const tx = requireAdmissionTx(scope, companyId);
return {
async saveReceipt(input) {
@@ -1561,6 +1562,36 @@ export function createFreshIssueRunWriter(
)).returning();
return receipt;
},
async readCommentCandidates({ issueId, agentId, handoffRunId, receiptScope, checkExecutionBlocker }) {
const [handoff] = handoffRunId ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, handoffRunId), eq(heartbeatRuns.companyId, companyId),
eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"),
ne(heartbeatRuns.agentId, agentId),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issueId)),
)) : [];
const candidates = receiptScope && (!checkExecutionBlocker || !(await getExecutionBlocker(tx, companyId, issueId)))
? await tx.select({
id: agentWakeupRequests.id,
reason: agentWakeupRequests.reason,
idempotencyKey: agentWakeupRequests.idempotencyKey,
payload: agentWakeupRequests.payload,
}).from(agentWakeupRequests).where(and(
eq(agentWakeupRequests.companyId, companyId),
inArray(agentWakeupRequests.agentId, handoff ? [agentId, handoff.agentId] : [agentId]),
eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
receiptScope.kind === "exact" ? eq(agentWakeupRequests.id, receiptScope.wakeId) : undefined,
)).orderBy(asc(agentWakeupRequests.requestedAt))
: [];
return { handoffAgentId: handoff?.agentId ?? null, candidates };
},
readUndeliveredLegacyCommentIds(issueId, agentId, commentIds) {
return undeliveredLegacyUserCommentIds(tx, companyId, issueId, agentId, commentIds);
},
notifyRunMilestones(issueId) {
return notifyChatRunMilestones(tx, { companyId, issueId });
},
async insertQueuedRun(seed) {
const [run] = await tx.insert(heartbeatRuns).values({
...seed, companyId, status: "queued",
@@ -1,5 +1,14 @@
import { isUuidLike } from "@paperclipai/shared";
import {
DEFERRED_WAKE_CONTEXT_KEY,
queuedCommentIdsFromRunContext,
queuedCommentIdsFromWakePayload,
withQueuedCommentIdsInRunContext,
withQueuedCommentIdsInWakePayload,
} from "../../../lib/queued-comment-context.js";
import { hasInteractionContinuationWakeContext, isInteractionResolutionWakePayload } from "../domain/context.js";
import { parseObject, readNonEmptyString } from "../domain/values.js";
import type { DurableWakeAdmissionReceipt } from "./ports.js";
import { readNonEmptyString } from "../domain/values.js";
type WakeReceiptSeed = {
agentId: string;
@@ -33,8 +42,24 @@ type SaveWakeReceipt =
| { kind: "insert"; seed: WakeReceiptSeed & Partial<DurableWakeAdmissionReceipt> }
| { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] };
export type FreshIssueRunWriter<Run> = {
type DeferredCommentCandidate = {
id: string;
reason: string | null;
idempotencyKey: string | null;
payload: unknown;
};
export type FreshIssueRunStore<Run> = {
saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>;
readCommentCandidates(input: {
issueId: string;
agentId: string;
handoffRunId: string | null;
receiptScope: { kind: "all" } | { kind: "exact"; wakeId: string } | null;
checkExecutionBlocker: boolean;
}): Promise<{ handoffAgentId: string | null; candidates: DeferredCommentCandidate[] }>;
readUndeliveredLegacyCommentIds(issueId: string, agentId: string, commentIds: string[]): Promise<string[]>;
notifyRunMilestones(issueId: string): Promise<void>;
insertQueuedRun(seed: QueuedRunSeed): Promise<Run>;
bindReceipts(input: {
wakeId: string;
@@ -44,53 +69,115 @@ export type FreshIssueRunWriter<Run> = {
}): Promise<void>;
};
/**
* Runs under heartbeat's existing issue lock. The caller validates admission,
* selects saved comments and records specialized continuation delivery between
* run creation and receipt binding.
*/
export function createFreshIssueRunAdmission<Run>(writer: FreshIssueRunWriter<Run>) {
return {
createReceipt(input: WakeReceiptSeed & {
durableReceipt?: DurableWakeAdmissionReceipt;
deferredChatReceiptId?: string;
}) {
const { durableReceipt, deferredChatReceiptId, ...seed } = input;
// A deferred durable chat wake keeps its original identity and source.
return writer.saveReceipt(deferredChatReceiptId
? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload }
: {
kind: "insert",
seed: durableReceipt
? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt }
: seed,
});
},
createRun(input: Omit<QueuedRunSeed,
"id" | "scopeKind" | "issueId" | "scheduledRetryAttempt" | "scheduledRetryReason"
> & {
explicitContinuationRunId: string | null;
reconciledSourceRunId: string | null;
reconciledRestoreRetryCount: number | null;
issueId: unknown;
}) {
const {
explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed
} = input;
const scopedIssueId = readNonEmptyString(issueId);
return writer.insertQueuedRun({
...seed,
...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}),
scopeKind: scopedIssueId ? "issue" : "company",
issueId: scopedIssueId,
retryOfRunId: reconciledSourceRunId || seed.retryOfRunId,
sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore,
...(reconciledRestoreRetryCount !== null ? {
scheduledRetryAttempt: reconciledRestoreRetryCount,
scheduledRetryReason: "transient_failure",
} : {}),
});
},
bindReceipts: writer.bindReceipts,
type FreshIssueRunInput = {
receipt: WakeReceiptSeed;
issue: { id: string; assigneeAgentId: string | null; isConversation: boolean };
durableReceipt?: DurableWakeAdmissionReceipt;
durableChatReceiptId?: string;
allowRunCoalescing?: boolean;
executionWaitRequestId?: string;
queuedCommentInterruptId?: string;
queuedCommentRequestId?: string;
boardRetryReceiptId?: string;
failedChatRetryReceiptId?: string;
ownsReconstructionContinuation: boolean;
contextSnapshot: Record<string, unknown>;
sessionIdBefore: string | null;
continuationAttempt: number;
explicitContinuationRunId: string | null;
retryOfRunId: string | null;
reconciledSourceRunId: string | null;
reconciledRestoreRetryCount: number | null;
resolveResponsibleUserId(): Promise<string>;
recordContinuationDelivery(runId: string): Promise<void>;
};
function isOrdinaryBoardComment(wake: DeferredCommentCandidate): boolean {
const payload = parseObject(wake.payload);
const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
// Durable chat and interaction wakes carry their own actor, source, and session.
return !wake.idempotencyKey?.startsWith("chat-inbound:") &&
!isInteractionResolutionWakePayload(payload) &&
!hasInteractionContinuationWakeContext(context) &&
["issue_commented", "issue_reopened_via_comment"].includes(String(context.wakeReason ?? wake.reason)) &&
queuedCommentIdsFromWakePayload(payload).length > 0;
}
/** Admit one new run and its saved inputs using the caller's existing issue lock. */
export function createFreshIssueRunAdmission<Run extends { id: string }>(store: FreshIssueRunStore<Run>) {
return async function admitFreshIssueRun(input: FreshIssueRunInput): Promise<Run | undefined> {
const { receipt, durableReceipt, durableChatReceiptId, executionWaitRequestId, issue, contextSnapshot } = input;
const { agentId, source, triggerDetail, reason, payload } = receipt;
// A deferred durable chat wake keeps its original identity and source.
const primary = await store.saveReceipt(durableChatReceiptId && durableChatReceiptId === executionWaitRequestId
? { kind: "promote", wakeId: durableChatReceiptId, agentId, payload }
: {
kind: "insert",
seed: durableReceipt ? { ...receipt, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } : receipt,
});
if (!primary) return undefined;
// Only a verified reassignment can bring a previous executor's saved comments.
const interruptedRunId = readNonEmptyString(contextSnapshot.interruptedRunId);
const handoffRunId = source === "assignment" && reason === "issue_assigned" &&
issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId)
? interruptedRunId : null;
// Chat keeps separate batches, but an exact resumed receipt must be consumed with its successor.
const canCoalesceComments = !durableChatReceiptId && !issue.isConversation && input.allowRunCoalescing !== false;
const { handoffAgentId, candidates } = await store.readCommentCandidates({
issueId: issue.id,
agentId,
handoffRunId,
receiptScope: canCoalesceComments || (executionWaitRequestId && input.boardRetryReceiptId)
? { kind: "all" }
: executionWaitRequestId ? { kind: "exact", wakeId: executionWaitRequestId } : null,
checkExecutionBlocker: !input.ownsReconstructionContinuation,
});
const adopted = candidates.filter((wake) => wake.id === executionWaitRequestId ||
wake.id === input.queuedCommentInterruptId || wake.id === input.queuedCommentRequestId ||
isOrdinaryBoardComment(wake));
const adoptedWakeIds = adopted.map((wake) => wake.id);
let commentIds = [...new Set([
...adopted.flatMap((wake) => queuedCommentIdsFromWakePayload(wake.payload)),
...queuedCommentIdsFromRunContext(contextSnapshot),
])];
if (input.queuedCommentRequestId || input.boardRetryReceiptId || handoffAgentId) {
commentIds = await store.readUndeliveredLegacyCommentIds(issue.id, agentId, commentIds);
}
await store.notifyRunMilestones(issue.id);
const responsibleUserId = await input.resolveResponsibleUserId();
const scopedIssueId = readNonEmptyString(contextSnapshot.issueId);
const run = await store.insertQueuedRun({
...(input.explicitContinuationRunId ? { id: input.explicitContinuationRunId } : {}),
agentId,
scopeKind: scopedIssueId ? "issue" : "company",
issueId: scopedIssueId,
invocationSource: source,
triggerDetail,
responsibleUserId,
wakeupRequestId: primary.id,
retryOfRunId: input.reconciledSourceRunId || input.retryOfRunId,
contextSnapshot: adopted.length ? withQueuedCommentIdsInRunContext(contextSnapshot, commentIds, adoptedWakeIds) : contextSnapshot,
sessionIdBefore: input.explicitContinuationRunId ? null : input.sessionIdBefore,
continuationAttempt: input.continuationAttempt,
...(input.reconciledRestoreRetryCount !== null ? {
scheduledRetryAttempt: input.reconciledRestoreRetryCount,
scheduledRetryReason: "transient_failure",
} : {}),
});
await input.recordContinuationDelivery(run.id);
await store.bindReceipts({
wakeId: primary.id,
runId: run.id,
failedChatRetryReceiptId: input.failedChatRetryReceiptId,
adopted: adopted.length ? {
wakeIds: adoptedWakeIds,
payload: withQueuedCommentIdsInWakePayload(payload, commentIds),
} : undefined,
});
return run;
};
}
@@ -1,4 +1,5 @@
import { readWakeChatConversationId } from "../../run-dispatch/index.js";
import { DEFERRED_WAKE_CONTEXT_KEY } from "../../../lib/queued-comment-context.js";
import { enrichPromotedWakeContext, wakeMustNotJoinActiveRun, withMergedWakeReasons } from "../domain/context.js";
import { QUEUED_MESSAGES_DELIVERED_REASON } from "../domain/policy.js";
import {
@@ -36,8 +37,6 @@ import type {
import type { PostCommitEffect, ReleaseOutcome } from "./types.js";
import { WakeQueueApplicationError } from "./types.js";
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair";
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([
"execution_review_requested",
@@ -114,7 +114,7 @@ export function hasRetainedExternalChatRetry(context: unknown): boolean {
(activeSource === "issue.interaction.respond" && active.externalChatContinuation === true));
}
function isInteractionResolutionWakePayload(payload: Record<string, unknown> | null | undefined): boolean {
export function isInteractionResolutionWakePayload(payload: Record<string, unknown> | null | undefined): boolean {
return readNonEmptyString(payload?.mutation) === "interaction";
}
+2 -2
View File
@@ -1,7 +1,7 @@
import type { Db } from "@paperclipai/db";
import {
createAdmissionTransactionScope as buildAdmissionTransactionScope,
createFreshIssueRunWriter,
createFreshIssueRunStore,
createPostgresWakeQueueAdapter,
createWakeAdmissionReader,
createWakeAdmissionWriter,
@@ -114,7 +114,7 @@ export type WakeQueue = ReturnType<typeof createWakeQueue>;
/** Uses the transaction that already owns the issue's admission lock. */
export function bindFreshIssueRunAdmission(companyId: string, tx: Db) {
return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId));
return createFreshIssueRunAdmission(createFreshIssueRunStore(buildAdmissionTransactionScope(companyId, tx), companyId));
}
/**
+49 -8
View File
@@ -7,11 +7,11 @@ import { eq, inArray, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js";
import { createFreshIssueRunStore, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js";
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js";
import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js";
import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js";
import { createWakeQueue } from "../../modules/wake-queue/index.js";
import { TransactionScope } from "../../modules/wake-queue/application/ports.js";
import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js";
import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js";
@@ -494,6 +494,49 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
return { company, agent, issue, wake, run };
}
it.each([true, false])("adopts only eligible saved comments from the exact reassignment source (%s)", async (matchingIssue) => {
const { user, company, agent, issue, deps, queue } = await waitingConversationFixture();
await db.update(issues).set({ conversationAgentId: null, conversationUserId: null,
conversationState: null, status: "todo" }).where(eq(issues.id, issue.id));
const [previousAgent] = await db.insert(agents).values({ companyId: company.id,
name: "Previous assignee", adapterType: "process" }).returning();
const [interruptedRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: previousAgent.id,
status: "cancelled", errorCode: "issue_reassigned", finishedAt: new Date(),
contextSnapshot: { issueId: matchingIssue ? issue.id : randomUUID() } }).returning();
const [earlier, later] = await db.insert(issueComments).values(["Earlier", "Later"].map(body => ({
companyId: company.id, issueId: issue.id, authorType: "user" as const, authorUserId: user.id, body,
}))).returning();
const [local, handoff, chat, interaction] = await db.insert(agentWakeupRequests).values([
{ agentId: agent.id, commentId: later.id, requestedAt: new Date(2) },
{ agentId: previousAgent.id, commentId: earlier.id, requestedAt: new Date(1) },
{ agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(3), idempotencyKey: `chat-inbound:${randomUUID()}` },
{ agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(4), mutation: "interaction" },
].map(({ agentId, commentId, requestedAt, idempotencyKey, mutation }) => ({
companyId: company.id, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution",
requestedAt, idempotencyKey, requestedByActorType: "user", requestedByActorId: user.id,
payload: withQueuedCommentIdsInWakePayload({ issueId: issue.id, mutation }, [commentId]),
}))).returning();
const run = await queue.enqueueWakeup(agent.id, {
source: "assignment", triggerDetail: "system", reason: "issue_assigned", requestedByActorType: "system",
payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, interruptedRunId: interruptedRun.id },
});
const commentIds = matchingIssue ? [earlier.id, later.id] : [later.id];
expect(run).toMatchObject({ status: "queued", responsibleUserId: user.id, invocationSource: "assignment",
contextSnapshot: { wakeCommentIds: commentIds, wakeCommentId: later.id,
queuedCommentSourceWakeIds: matchingIssue ? [handoff.id, local.id] : [local.id] } });
const receipts = new Map((await db.select().from(agentWakeupRequests)).map(wake => [wake.id, wake]));
expect(receipts.get(run!.wakeupRequestId!)).toMatchObject({ runId: run!.id, source: "assignment",
requestedByActorType: "system", payload: { _paperclipWakeContext: { wakeCommentIds: commentIds } } });
expect(receipts.get(local.id)).toMatchObject({ status: "coalesced", runId: run!.id, requestedByActorId: user.id });
expect(receipts.get(handoff.id)).toMatchObject(matchingIssue
? { status: "coalesced", runId: run!.id, agentId: previousAgent.id }
: { status: "deferred_issue_execution", runId: null });
for (const receipt of [chat, interaction]) expect(receipts.get(receipt.id)).toEqual(receipt);
expect(deps.executeRun).not.toHaveBeenCalled();
});
it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => {
const local = await fixture("queued");
const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning();
@@ -510,13 +553,11 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
}).returning();
await db.transaction(async (tx) => {
const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db);
expect(await admission.createReceipt({
deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation",
triggerDetail: null, reason: null, payload: null, requestedByActorType: null,
requestedByActorId: null, idempotencyKey: null,
const store = createFreshIssueRunStore(TransactionScope.create(local.company.id, tx), local.company.id);
expect(await store.saveReceipt({
kind: "promote", wakeId: localDeferredWake.id, agentId: foreignAgent.id, payload: null,
})).toBeUndefined();
await admission.bindReceipts({
await store.bindReceipts({
wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id,
adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } },
});
+42 -127
View File
@@ -144,11 +144,12 @@ import { isUniqueViolation } from "../../db-errors.js";
import { publishLiveEvent } from "../live-events.js";
import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js";
import {
DEFERRED_WAKE_CONTEXT_KEY,
queuedCommentIdsFromWakePayload,
queuedCommentIdsFromRunContext,
withQueuedCommentIdsInWakePayload,
withQueuedCommentIdsInRunContext,
} from "../issue-queued-comment-queue.js";
} from "../../lib/queued-comment-context.js";
import { parseObject } from "../../adapters/utils.js";
import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js";
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
@@ -277,7 +278,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
export { DEFERRED_WAKE_CONTEXT_KEY } from "../../lib/queued-comment-context.js";
export const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [
"queued",
@@ -5069,141 +5070,55 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}, issueActivityPublications);
}
const freshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db);
const wakeupRequest = await freshIssueRun.createReceipt({
const admitFreshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db);
const newRun = await admitFreshIssueRun({
receipt: {
agentId,
source,
triggerDetail,
reason,
payload,
requestedByActorType: opts.requestedByActorType ?? null,
requestedByActorId: opts.requestedByActorId ?? null,
idempotencyKey: opts.idempotencyKey ?? null,
},
issue: { id: issue.id, assigneeAgentId: issue.assigneeAgentId, isConversation: isConversation(issue) },
durableReceipt: receiptRequest,
deferredChatReceiptId: durableRequest && executionWaitRequestId === durableRequest.id
? durableRequest.id : undefined,
agentId,
source,
triggerDetail,
reason,
payload,
requestedByActorType: opts.requestedByActorType ?? null,
requestedByActorId: opts.requestedByActorId ?? null,
idempotencyKey: opts.idempotencyKey ?? null,
});
if (!wakeupRequest) return { kind: "deferred" as const };
// A handoff changes the executor, not the owner of saved user input.
// Validate its exact stopped source while the issue row is locked;
// unrelated agents and dedicated continuations keep their own wakes.
const interruptedRunId = readNonEmptyString(enrichedContextSnapshot.interruptedRunId);
const handoffSource = source === "assignment" && reason === "issue_assigned" &&
issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId)
? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, interruptedRunId), eq(heartbeatRuns.companyId, issue.companyId),
eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"),
ne(heartbeatRuns.agentId, agentId),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`,
or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)),
)).then(rows => rows[0] ?? null)
: null;
// External input owns an exact authorized source batch. Board input
// must retain its separate receipt, including during normal resume.
const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false;
// A resumed receipt already passed admission under this issue lock.
// Consume it with its successor even when chat keeps other messages
// in separate turns, or finalization will deliver it a second time.
const pendingComments =
(executionWaitRequestId || canCoalesceComments) &&
(ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id)))
? await tx
.select()
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, issue.companyId),
inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]),
eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`,
// Chat Retry already pins its complete authorized source
// batch. Keep separate board input for its own admission.
canCoalesceComments || (opts.taskRetryRequestId && !failedChatRetry)
? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!),
),
)
.orderBy(asc(agentWakeupRequests.requestedAt))
: [];
const adoptedComments = pendingComments.filter((wake) => {
if (wake.id === executionWaitRequestId) return true;
if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true;
const deferredPayload = parseObject(wake.payload);
const deferredContext = parseObject(
deferredPayload[DEFERRED_WAKE_CONTEXT_KEY],
);
// Dedicated interaction wakes carry their own source and session
// contract. ID-only adoption must not erase that continuation.
return (
// Durable chat work must keep its receipt, actor, source, and
// session contract through normal promotion and authorization.
!wake.idempotencyKey?.startsWith("chat-inbound:") &&
!isInteractionResolutionWakePayload(deferredPayload) &&
!hasInteractionContinuationWakeContext(deferredContext) &&
["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) &&
queuedCommentIdsFromWakePayload(wake.payload).length > 0
);
});
let adoptedCommentIds = [
...new Set([
...adoptedComments.flatMap((wake) =>
queuedCommentIdsFromWakePayload(wake.payload),
),
...queuedCommentIdsFromRunContext(enrichedContextSnapshot),
]),
];
if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) {
adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db,
agent.companyId, issueId, agentId, adoptedCommentIds);
}
await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined });
const newRun = await freshIssueRun.createRun({
durableChatReceiptId: durableRequest?.id,
allowRunCoalescing: opts.allowRunCoalescing,
executionWaitRequestId,
queuedCommentInterruptId: opts.queuedCommentInterruptId,
queuedCommentRequestId: opts.queuedCommentRequestId,
boardRetryReceiptId: failedChatRetry ? undefined : opts.taskRetryRequestId,
failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined,
ownsReconstructionContinuation: Boolean(ownsReconstructionContinuation),
explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null,
agentId,
issueId: enrichedContextSnapshot.issueId,
invocationSource: source,
triggerDetail,
responsibleUserId: await resolveQueuedResponsibleUserId(),
wakeupRequestId: wakeupRequest.id,
retryOfRunId: failedChatRetry
? durableRequest!.failedRunRetry!.failedRunId
: opts.failedRunId ?? automaticParentRunId,
contextSnapshot: adoptedComments.length
? withQueuedCommentIdsInRunContext(
enrichedContextSnapshot,
adoptedCommentIds,
adoptedComments.map((wake) => wake.id),
)
: enrichedContextSnapshot,
contextSnapshot: enrichedContextSnapshot,
sessionIdBefore: sessionBefore,
continuationAttempt,
reconciledSourceRunId,
reconciledRestoreRetryCount,
resolveResponsibleUserId: resolveQueuedResponsibleUserId,
async recordContinuationDelivery(runId) {
if (ownsReconstructionContinuation) {
await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({
continuationDelivery: "delivered", continuationRunId: runId,
})}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId),
eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)),
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`));
}
if (continuationToDeliver)
await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id,
runId, continuation: continuationToDeliver });
else if (reconciledSourceRunId)
await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id,
runId, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId });
},
});
if (ownsReconstructionContinuation) {
await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({
continuationDelivery: "delivered", continuationRunId: newRun.id,
})}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId),
eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)),
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`));
}
if (continuationToDeliver)
await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id,
continuation: continuationToDeliver });
else if (reconciledSourceRunId)
await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id,
runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId });
await freshIssueRun.bindReceipts({
wakeId: wakeupRequest.id,
runId: newRun.id,
failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined,
adopted: adoptedComments.length ? {
wakeIds: adoptedComments.map((wake) => wake.id),
payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds),
} : undefined,
});
if (!newRun) return { kind: "deferred" as const };
// executionRunId is NOT stamped here (enqueueWakeup queues the run but
// doesn't start it). It will be stamped in claimQueuedRun() once the run
@@ -1,8 +1,12 @@
import { createHash } from "node:crypto";
import type { IssueComment, IssueQueuedCommentQueue } from "@paperclipai/shared";
const QUEUE_CONTEXT_KEY = "_paperclipWakeContext";
const QUEUE_IDS_KEY = "wakeCommentIds";
export {
queuedCommentIdsFromRunContext,
queuedCommentIdsFromWakePayload,
withQueuedCommentIdsInRunContext,
withQueuedCommentIdsInWakePayload,
} from "../lib/queued-comment-context.js";
function record(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
@@ -10,51 +14,6 @@ function record(value: unknown): Record<string, unknown> {
: {};
}
function uniqueIds(value: unknown): string[] {
if (!Array.isArray(value)) return [];
const seen = new Set<string>();
return value.flatMap((candidate) => {
if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return [];
seen.add(candidate);
return [candidate];
});
}
export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] {
const payload = record(payloadValue);
const context = record(payload[QUEUE_CONTEXT_KEY]);
return uniqueIds(context[QUEUE_IDS_KEY]);
}
export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] {
return uniqueIds(record(contextValue)[QUEUE_IDS_KEY]);
}
export function withQueuedCommentIdsInWakePayload(
payloadValue: unknown,
ids: string[],
options: { preserveSource?: boolean } = {},
): Record<string, unknown> {
const payload = { ...record(payloadValue) };
const context = { ...record(payload[QUEUE_CONTEXT_KEY]) };
if (ids.length > 0) {
const latestId = ids[ids.length - 1]!;
context[QUEUE_IDS_KEY] = ids;
if (!options.preserveSource) {
context.wakeCommentId = latestId;
context.commentId = latestId;
payload.commentId = latestId;
}
} else {
delete context[QUEUE_IDS_KEY];
delete context.wakeCommentId;
delete context.commentId;
delete payload.commentId;
}
payload[QUEUE_CONTEXT_KEY] = context;
return payload;
}
/**
* Fingerprints one queued-comment queue: the wake id plus every comment's id
* and last-updated time. A mutation that changes the queue changes this
@@ -76,41 +35,6 @@ export function queuedCommentQueueRevision(input: {
.slice(0, 32);
}
export function withQueuedCommentIdsInRunContext(
contextValue: unknown,
ids: string[],
sourceWakeIds?: string[],
): Record<string, unknown> {
const context = { ...record(contextValue) };
if (sourceWakeIds) {
context.queuedCommentSourceWakeIds = uniqueIds([
...uniqueIds(context.queuedCommentSourceWakeIds),
...sourceWakeIds,
]);
}
if (ids.length > 0) {
const latestId = ids[ids.length - 1]!;
context[QUEUE_IDS_KEY] = ids;
context.wakeCommentId = latestId;
context.commentId = latestId;
} else {
delete context[QUEUE_IDS_KEY];
delete context.wakeCommentId;
delete context.commentId;
}
// These projections are generated immediately before dispatch. Any queue
// mutation must force them to be rebuilt from the canonical comment ids.
delete context.paperclipWake;
delete context.paperclipWakeComment;
delete context.paperclipTaskMarkdown;
delete context.paperclipTaskMarkdownCompact;
delete context.paperclipTaskMarkdownAssignment;
delete context.paperclipTaskMarkdownAssignmentCompact;
delete context.paperclipTurnContext;
return context;
}
export type QueuedCommentQueueProtocol = "paperclip_runner_v1" | "legacy";
export type QueuedCommentQueueSteeringDecision =