From c957f8529c5747a65dca6b63b8a28241d35f258b Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Wed, 18 Nov 2026 17:02:52 -0800 Subject: [PATCH] Admit saved task comments with the fresh issue run Co-Authored-By: Paperclip --- doc/heartbeat-internals.md | 6 +- server/src/lib/queued-comment-context.ts | 88 +++++++++ .../modules/run-dispatch/adapters/postgres.ts | 2 +- .../run-dispatch/domain/wake-context.ts | 4 +- .../modules/wake-queue/adapters/postgres.ts | 45 ++++- .../wake-queue/application/fresh-issue-run.ts | 187 +++++++++++++----- .../wake-queue/application/use-cases.ts | 3 +- .../src/modules/wake-queue/domain/context.ts | 2 +- server/src/modules/wake-queue/index.ts | 4 +- server/src/services/heartbeat/queue.test.ts | 57 +++++- server/src/services/heartbeat/queue.ts | 169 ++++------------ .../services/issue-queued-comment-queue.ts | 88 +-------- 12 files changed, 372 insertions(+), 283 deletions(-) create mode 100644 server/src/lib/queued-comment-context.ts diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index ba0ce713ae4..ffe8d1dfee4 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -316,8 +316,10 @@ Keep recovery policy changes separate from retry scheduling and execution change Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`. It owns wake coalescing and batching, queued-run claims, daily heartbeat caps, concurrency and priority checks, timer admission, and native status wake intents. -The wake-queue module creates fresh task runs and binds their receipts inside -heartbeat's issue admission transaction. +The wake-queue module admits fresh task runs inside heartbeat's issue admission +transaction. It selects eligible saved comment receipts in request order and +binds them to the new run. Heartbeat supplies responsible-user resolution and +records any specialized continuation delivery before those receipts are bound. `createHeartbeatQueue` binds these operations without querying or starting work. The service supplies lifecycle effects, execution callbacks, worktree gates, and its process-wide execution and wakeup promise sets. Forwarding callbacks preserve diff --git a/server/src/lib/queued-comment-context.ts b/server/src/lib/queued-comment-context.ts new file mode 100644 index 00000000000..41148b9006e --- /dev/null +++ b/server/src/lib/queued-comment-context.ts @@ -0,0 +1,88 @@ +export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; + +function parseObject(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function uniqueIds(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const seen = new Set(); + return value.flatMap((candidate) => { + if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return []; + seen.add(candidate); + return [candidate]; + }); +} + +export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] { + const payload = parseObject(payloadValue); + const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + return uniqueIds(context[WAKE_COMMENT_IDS_KEY]); +} + +export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] { + return uniqueIds(parseObject(contextValue)[WAKE_COMMENT_IDS_KEY]); +} + +export function withQueuedCommentIdsInWakePayload( + payloadValue: unknown, + ids: string[], + options: { preserveSource?: boolean } = {}, +): Record { + const payload = { ...parseObject(payloadValue) }; + const context = { ...parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]) }; + if (ids.length > 0) { + const latestId = ids[ids.length - 1]!; + context[WAKE_COMMENT_IDS_KEY] = ids; + if (!options.preserveSource) { + context.wakeCommentId = latestId; + context.commentId = latestId; + payload.commentId = latestId; + } + } else { + delete context[WAKE_COMMENT_IDS_KEY]; + delete context.wakeCommentId; + delete context.commentId; + delete payload.commentId; + } + payload[DEFERRED_WAKE_CONTEXT_KEY] = context; + return payload; +} + +export function withQueuedCommentIdsInRunContext( + contextValue: unknown, + ids: string[], + sourceWakeIds?: string[], +): Record { + const context = { ...parseObject(contextValue) }; + if (sourceWakeIds) { + context.queuedCommentSourceWakeIds = uniqueIds([ + ...uniqueIds(context.queuedCommentSourceWakeIds), + ...sourceWakeIds, + ]); + } + if (ids.length > 0) { + const latestId = ids[ids.length - 1]!; + context[WAKE_COMMENT_IDS_KEY] = ids; + context.wakeCommentId = latestId; + context.commentId = latestId; + } else { + delete context[WAKE_COMMENT_IDS_KEY]; + delete context.wakeCommentId; + delete context.commentId; + } + + // These projections are generated immediately before dispatch. Any queue + // mutation must force them to be rebuilt from the canonical comment ids. + delete context.paperclipWake; + delete context.paperclipWakeComment; + delete context.paperclipTaskMarkdown; + delete context.paperclipTaskMarkdownCompact; + delete context.paperclipTaskMarkdownAssignment; + delete context.paperclipTaskMarkdownAssignmentCompact; + delete context.paperclipTurnContext; + return context; +} diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 592a8d1517d..810d1494915 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -5,7 +5,7 @@ import { readNativeWorkspaceSyncReference } from "../../../services/native-runti import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; import { isAgentDirectoryPreparationWait } from "../../../services/pre-dispatch-execution-evidence.js"; import { isUserCommentContinuationReason } from "../../../services/explicit-native-continuation.js"; -import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js"; +import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../lib/queued-comment-context.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "../../../services/cancelled-workspace-preservation-notifications.js"; diff --git a/server/src/modules/run-dispatch/domain/wake-context.ts b/server/src/modules/run-dispatch/domain/wake-context.ts index 18c7716359e..26cddce4516 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.ts @@ -5,6 +5,9 @@ // independently maintained copy in either file could silently drift out of // sync with the other and change only one of the two gates that read it. +import { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js"; +export { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js"; + function parseObject(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -30,7 +33,6 @@ export function isPreparationWaitRetryReason(reason: string | null | undefined): } export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry"; export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry"; -export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; export const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set([ "accepted", "answered", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index a70fa7b926a..7caa80b5f36 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -16,7 +16,7 @@ import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction- import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js"; -import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, ne, notInArray, or, sql } from "drizzle-orm"; import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js"; import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js"; @@ -53,13 +53,15 @@ import { executionRetryAccounting } from "../../../services/execution-recovery-a import { issueService } from "../../../services/issues.js"; import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; +import { undeliveredLegacyUserCommentIds } from "../../../services/explicit-native-continuation.js"; import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; import { - queuedCommentIdsFromWakePayload, + DEFERRED_WAKE_CONTEXT_KEY, queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, withQueuedCommentIdsInWakePayload, -} from "../../../services/issue-queued-comment-queue.js"; +} from "../../../lib/queued-comment-context.js"; import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds, readWakeChatConversationId } from "../../run-dispatch/index.js"; import { deferredWakeHoldsNoOtherConversation, otherConversationCommentIds, splitOtherConversationMessages } from "./chat-conversation-batch.js"; import { hasInteractionContinuationWakeContext } from "../domain/context.js"; @@ -85,13 +87,12 @@ import type { WakeQueueHost, WakeQueueTransaction, } from "../application/ports.js"; -import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js"; +import type { FreshIssueRunStore } from "../application/fresh-issue-run.js"; import type { RunSummary } from "../application/types.js"; const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; /** Wake outcome for a deferred chat wake whose news an earlier reply already gave. */ export const ALREADY_REPORTED_WAKE_REASON = "Already reported by an earlier reply in this conversation"; -const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; @@ -1537,10 +1538,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd }; } -export function createFreshIssueRunWriter( +export function createFreshIssueRunStore( scope: TransactionScope, companyId: string, -): FreshIssueRunWriter { +): FreshIssueRunStore { const tx = requireAdmissionTx(scope, companyId); return { async saveReceipt(input) { @@ -1561,6 +1562,36 @@ export function createFreshIssueRunWriter( )).returning(); return receipt; }, + async readCommentCandidates({ issueId, agentId, handoffRunId, receiptScope, checkExecutionBlocker }) { + const [handoff] = handoffRunId ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, handoffRunId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"), + ne(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issueId)), + )) : []; + const candidates = receiptScope && (!checkExecutionBlocker || !(await getExecutionBlocker(tx, companyId, issueId))) + ? await tx.select({ + id: agentWakeupRequests.id, + reason: agentWakeupRequests.reason, + idempotencyKey: agentWakeupRequests.idempotencyKey, + payload: agentWakeupRequests.payload, + }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.agentId, handoff ? [agentId, handoff.agentId] : [agentId]), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + receiptScope.kind === "exact" ? eq(agentWakeupRequests.id, receiptScope.wakeId) : undefined, + )).orderBy(asc(agentWakeupRequests.requestedAt)) + : []; + return { handoffAgentId: handoff?.agentId ?? null, candidates }; + }, + readUndeliveredLegacyCommentIds(issueId, agentId, commentIds) { + return undeliveredLegacyUserCommentIds(tx, companyId, issueId, agentId, commentIds); + }, + notifyRunMilestones(issueId) { + return notifyChatRunMilestones(tx, { companyId, issueId }); + }, async insertQueuedRun(seed) { const [run] = await tx.insert(heartbeatRuns).values({ ...seed, companyId, status: "queued", diff --git a/server/src/modules/wake-queue/application/fresh-issue-run.ts b/server/src/modules/wake-queue/application/fresh-issue-run.ts index 52bebb8a082..a5c7da4a4f3 100644 --- a/server/src/modules/wake-queue/application/fresh-issue-run.ts +++ b/server/src/modules/wake-queue/application/fresh-issue-run.ts @@ -1,5 +1,14 @@ +import { isUuidLike } from "@paperclipai/shared"; +import { + DEFERRED_WAKE_CONTEXT_KEY, + queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInRunContext, + withQueuedCommentIdsInWakePayload, +} from "../../../lib/queued-comment-context.js"; +import { hasInteractionContinuationWakeContext, isInteractionResolutionWakePayload } from "../domain/context.js"; +import { parseObject, readNonEmptyString } from "../domain/values.js"; import type { DurableWakeAdmissionReceipt } from "./ports.js"; -import { readNonEmptyString } from "../domain/values.js"; type WakeReceiptSeed = { agentId: string; @@ -33,8 +42,24 @@ type SaveWakeReceipt = | { kind: "insert"; seed: WakeReceiptSeed & Partial } | { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] }; -export type FreshIssueRunWriter = { +type DeferredCommentCandidate = { + id: string; + reason: string | null; + idempotencyKey: string | null; + payload: unknown; +}; + +export type FreshIssueRunStore = { saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>; + readCommentCandidates(input: { + issueId: string; + agentId: string; + handoffRunId: string | null; + receiptScope: { kind: "all" } | { kind: "exact"; wakeId: string } | null; + checkExecutionBlocker: boolean; + }): Promise<{ handoffAgentId: string | null; candidates: DeferredCommentCandidate[] }>; + readUndeliveredLegacyCommentIds(issueId: string, agentId: string, commentIds: string[]): Promise; + notifyRunMilestones(issueId: string): Promise; insertQueuedRun(seed: QueuedRunSeed): Promise; bindReceipts(input: { wakeId: string; @@ -44,53 +69,115 @@ export type FreshIssueRunWriter = { }): Promise; }; -/** - * Runs under heartbeat's existing issue lock. The caller validates admission, - * selects saved comments and records specialized continuation delivery between - * run creation and receipt binding. - */ -export function createFreshIssueRunAdmission(writer: FreshIssueRunWriter) { - return { - createReceipt(input: WakeReceiptSeed & { - durableReceipt?: DurableWakeAdmissionReceipt; - deferredChatReceiptId?: string; - }) { - const { durableReceipt, deferredChatReceiptId, ...seed } = input; - // A deferred durable chat wake keeps its original identity and source. - return writer.saveReceipt(deferredChatReceiptId - ? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload } - : { - kind: "insert", - seed: durableReceipt - ? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } - : seed, - }); - }, - createRun(input: Omit & { - explicitContinuationRunId: string | null; - reconciledSourceRunId: string | null; - reconciledRestoreRetryCount: number | null; - issueId: unknown; - }) { - const { - explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed - } = input; - const scopedIssueId = readNonEmptyString(issueId); - return writer.insertQueuedRun({ - ...seed, - ...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}), - scopeKind: scopedIssueId ? "issue" : "company", - issueId: scopedIssueId, - retryOfRunId: reconciledSourceRunId || seed.retryOfRunId, - sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore, - ...(reconciledRestoreRetryCount !== null ? { - scheduledRetryAttempt: reconciledRestoreRetryCount, - scheduledRetryReason: "transient_failure", - } : {}), - }); - }, - bindReceipts: writer.bindReceipts, +type FreshIssueRunInput = { + receipt: WakeReceiptSeed; + issue: { id: string; assigneeAgentId: string | null; isConversation: boolean }; + durableReceipt?: DurableWakeAdmissionReceipt; + durableChatReceiptId?: string; + allowRunCoalescing?: boolean; + executionWaitRequestId?: string; + queuedCommentInterruptId?: string; + queuedCommentRequestId?: string; + boardRetryReceiptId?: string; + failedChatRetryReceiptId?: string; + ownsReconstructionContinuation: boolean; + contextSnapshot: Record; + sessionIdBefore: string | null; + continuationAttempt: number; + explicitContinuationRunId: string | null; + retryOfRunId: string | null; + reconciledSourceRunId: string | null; + reconciledRestoreRetryCount: number | null; + resolveResponsibleUserId(): Promise; + recordContinuationDelivery(runId: string): Promise; +}; + +function isOrdinaryBoardComment(wake: DeferredCommentCandidate): boolean { + const payload = parseObject(wake.payload); + const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + // Durable chat and interaction wakes carry their own actor, source, and session. + return !wake.idempotencyKey?.startsWith("chat-inbound:") && + !isInteractionResolutionWakePayload(payload) && + !hasInteractionContinuationWakeContext(context) && + ["issue_commented", "issue_reopened_via_comment"].includes(String(context.wakeReason ?? wake.reason)) && + queuedCommentIdsFromWakePayload(payload).length > 0; +} + +/** Admit one new run and its saved inputs using the caller's existing issue lock. */ +export function createFreshIssueRunAdmission(store: FreshIssueRunStore) { + return async function admitFreshIssueRun(input: FreshIssueRunInput): Promise { + const { receipt, durableReceipt, durableChatReceiptId, executionWaitRequestId, issue, contextSnapshot } = input; + const { agentId, source, triggerDetail, reason, payload } = receipt; + + // A deferred durable chat wake keeps its original identity and source. + const primary = await store.saveReceipt(durableChatReceiptId && durableChatReceiptId === executionWaitRequestId + ? { kind: "promote", wakeId: durableChatReceiptId, agentId, payload } + : { + kind: "insert", + seed: durableReceipt ? { ...receipt, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } : receipt, + }); + if (!primary) return undefined; + + // Only a verified reassignment can bring a previous executor's saved comments. + const interruptedRunId = readNonEmptyString(contextSnapshot.interruptedRunId); + const handoffRunId = source === "assignment" && reason === "issue_assigned" && + issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId) + ? interruptedRunId : null; + + // Chat keeps separate batches, but an exact resumed receipt must be consumed with its successor. + const canCoalesceComments = !durableChatReceiptId && !issue.isConversation && input.allowRunCoalescing !== false; + const { handoffAgentId, candidates } = await store.readCommentCandidates({ + issueId: issue.id, + agentId, + handoffRunId, + receiptScope: canCoalesceComments || (executionWaitRequestId && input.boardRetryReceiptId) + ? { kind: "all" } + : executionWaitRequestId ? { kind: "exact", wakeId: executionWaitRequestId } : null, + checkExecutionBlocker: !input.ownsReconstructionContinuation, + }); + const adopted = candidates.filter((wake) => wake.id === executionWaitRequestId || + wake.id === input.queuedCommentInterruptId || wake.id === input.queuedCommentRequestId || + isOrdinaryBoardComment(wake)); + const adoptedWakeIds = adopted.map((wake) => wake.id); + let commentIds = [...new Set([ + ...adopted.flatMap((wake) => queuedCommentIdsFromWakePayload(wake.payload)), + ...queuedCommentIdsFromRunContext(contextSnapshot), + ])]; + if (input.queuedCommentRequestId || input.boardRetryReceiptId || handoffAgentId) { + commentIds = await store.readUndeliveredLegacyCommentIds(issue.id, agentId, commentIds); + } + + await store.notifyRunMilestones(issue.id); + const responsibleUserId = await input.resolveResponsibleUserId(); + const scopedIssueId = readNonEmptyString(contextSnapshot.issueId); + const run = await store.insertQueuedRun({ + ...(input.explicitContinuationRunId ? { id: input.explicitContinuationRunId } : {}), + agentId, + scopeKind: scopedIssueId ? "issue" : "company", + issueId: scopedIssueId, + invocationSource: source, + triggerDetail, + responsibleUserId, + wakeupRequestId: primary.id, + retryOfRunId: input.reconciledSourceRunId || input.retryOfRunId, + contextSnapshot: adopted.length ? withQueuedCommentIdsInRunContext(contextSnapshot, commentIds, adoptedWakeIds) : contextSnapshot, + sessionIdBefore: input.explicitContinuationRunId ? null : input.sessionIdBefore, + continuationAttempt: input.continuationAttempt, + ...(input.reconciledRestoreRetryCount !== null ? { + scheduledRetryAttempt: input.reconciledRestoreRetryCount, + scheduledRetryReason: "transient_failure", + } : {}), + }); + await input.recordContinuationDelivery(run.id); + await store.bindReceipts({ + wakeId: primary.id, + runId: run.id, + failedChatRetryReceiptId: input.failedChatRetryReceiptId, + adopted: adopted.length ? { + wakeIds: adoptedWakeIds, + payload: withQueuedCommentIdsInWakePayload(payload, commentIds), + } : undefined, + }); + return run; }; } diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts index fb6a31c0f61..ccd0817f78b 100644 --- a/server/src/modules/wake-queue/application/use-cases.ts +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -1,4 +1,5 @@ import { readWakeChatConversationId } from "../../run-dispatch/index.js"; +import { DEFERRED_WAKE_CONTEXT_KEY } from "../../../lib/queued-comment-context.js"; import { enrichPromotedWakeContext, wakeMustNotJoinActiveRun, withMergedWakeReasons } from "../domain/context.js"; import { QUEUED_MESSAGES_DELIVERED_REASON } from "../domain/policy.js"; import { @@ -36,8 +37,6 @@ import type { import type { PostCommitEffect, ReleaseOutcome } from "./types.js"; import { WakeQueueApplicationError } from "./types.js"; -const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; - const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair"; const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([ "execution_review_requested", diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts index 51a7c6e91f7..20607eba742 100644 --- a/server/src/modules/wake-queue/domain/context.ts +++ b/server/src/modules/wake-queue/domain/context.ts @@ -114,7 +114,7 @@ export function hasRetainedExternalChatRetry(context: unknown): boolean { (activeSource === "issue.interaction.respond" && active.externalChatContinuation === true)); } -function isInteractionResolutionWakePayload(payload: Record | null | undefined): boolean { +export function isInteractionResolutionWakePayload(payload: Record | null | undefined): boolean { return readNonEmptyString(payload?.mutation) === "interaction"; } diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 33fd109e102..2ad2b1247ab 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -1,7 +1,7 @@ import type { Db } from "@paperclipai/db"; import { createAdmissionTransactionScope as buildAdmissionTransactionScope, - createFreshIssueRunWriter, + createFreshIssueRunStore, createPostgresWakeQueueAdapter, createWakeAdmissionReader, createWakeAdmissionWriter, @@ -114,7 +114,7 @@ export type WakeQueue = ReturnType; /** Uses the transaction that already owns the issue's admission lock. */ export function bindFreshIssueRunAdmission(companyId: string, tx: Db) { - return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId)); + return createFreshIssueRunAdmission(createFreshIssueRunStore(buildAdmissionTransactionScope(companyId, tx), companyId)); } /** diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index a67dd4cc4a9..8808e4871e9 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -7,11 +7,11 @@ import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; -import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js"; +import { createFreshIssueRunStore, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; -import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js"; +import { createWakeQueue } from "../../modules/wake-queue/index.js"; import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; @@ -494,6 +494,49 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { company, agent, issue, wake, run }; } + it.each([true, false])("adopts only eligible saved comments from the exact reassignment source (%s)", async (matchingIssue) => { + const { user, company, agent, issue, deps, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, + conversationState: null, status: "todo" }).where(eq(issues.id, issue.id)); + const [previousAgent] = await db.insert(agents).values({ companyId: company.id, + name: "Previous assignee", adapterType: "process" }).returning(); + const [interruptedRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: previousAgent.id, + status: "cancelled", errorCode: "issue_reassigned", finishedAt: new Date(), + contextSnapshot: { issueId: matchingIssue ? issue.id : randomUUID() } }).returning(); + const [earlier, later] = await db.insert(issueComments).values(["Earlier", "Later"].map(body => ({ + companyId: company.id, issueId: issue.id, authorType: "user" as const, authorUserId: user.id, body, + }))).returning(); + const [local, handoff, chat, interaction] = await db.insert(agentWakeupRequests).values([ + { agentId: agent.id, commentId: later.id, requestedAt: new Date(2) }, + { agentId: previousAgent.id, commentId: earlier.id, requestedAt: new Date(1) }, + { agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(3), idempotencyKey: `chat-inbound:${randomUUID()}` }, + { agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(4), mutation: "interaction" }, + ].map(({ agentId, commentId, requestedAt, idempotencyKey, mutation }) => ({ + companyId: company.id, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution", + requestedAt, idempotencyKey, requestedByActorType: "user", requestedByActorId: user.id, + payload: withQueuedCommentIdsInWakePayload({ issueId: issue.id, mutation }, [commentId]), + }))).returning(); + + const run = await queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "issue_assigned", requestedByActorType: "system", + payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, interruptedRunId: interruptedRun.id }, + }); + + const commentIds = matchingIssue ? [earlier.id, later.id] : [later.id]; + expect(run).toMatchObject({ status: "queued", responsibleUserId: user.id, invocationSource: "assignment", + contextSnapshot: { wakeCommentIds: commentIds, wakeCommentId: later.id, + queuedCommentSourceWakeIds: matchingIssue ? [handoff.id, local.id] : [local.id] } }); + const receipts = new Map((await db.select().from(agentWakeupRequests)).map(wake => [wake.id, wake])); + expect(receipts.get(run!.wakeupRequestId!)).toMatchObject({ runId: run!.id, source: "assignment", + requestedByActorType: "system", payload: { _paperclipWakeContext: { wakeCommentIds: commentIds } } }); + expect(receipts.get(local.id)).toMatchObject({ status: "coalesced", runId: run!.id, requestedByActorId: user.id }); + expect(receipts.get(handoff.id)).toMatchObject(matchingIssue + ? { status: "coalesced", runId: run!.id, agentId: previousAgent.id } + : { status: "deferred_issue_execution", runId: null }); + for (const receipt of [chat, interaction]) expect(receipts.get(receipt.id)).toEqual(receipt); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => { const local = await fixture("queued"); const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); @@ -510,13 +553,11 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { }).returning(); await db.transaction(async (tx) => { - const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db); - expect(await admission.createReceipt({ - deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation", - triggerDetail: null, reason: null, payload: null, requestedByActorType: null, - requestedByActorId: null, idempotencyKey: null, + const store = createFreshIssueRunStore(TransactionScope.create(local.company.id, tx), local.company.id); + expect(await store.saveReceipt({ + kind: "promote", wakeId: localDeferredWake.id, agentId: foreignAgent.id, payload: null, })).toBeUndefined(); - await admission.bindReceipts({ + await store.bindReceipts({ wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id, adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } }, }); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 79da5a3f8c7..99154574820 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -144,11 +144,12 @@ import { isUniqueViolation } from "../../db-errors.js"; import { publishLiveEvent } from "../live-events.js"; import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js"; import { + DEFERRED_WAKE_CONTEXT_KEY, queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext, -} from "../issue-queued-comment-queue.js"; +} from "../../lib/queued-comment-context.js"; import { parseObject } from "../../adapters/utils.js"; import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; @@ -277,7 +278,7 @@ export interface HeartbeatQueueDependencies extends Pick>'issueId' = ${issue.id}`, - or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), - )).then(rows => rows[0] ?? null) - : null; - // External input owns an exact authorized source batch. Board input - // must retain its separate receipt, including during normal resume. - const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false; - // A resumed receipt already passed admission under this issue lock. - // Consume it with its successor even when chat keeps other messages - // in separate turns, or finalization will deliver it a second time. - const pendingComments = - (executionWaitRequestId || canCoalesceComments) && - (ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id))) - ? await tx - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, - // Chat Retry already pins its complete authorized source - // batch. Keep separate board input for its own admission. - canCoalesceComments || (opts.taskRetryRequestId && !failedChatRetry) - ? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!), - ), - ) - .orderBy(asc(agentWakeupRequests.requestedAt)) - : []; - const adoptedComments = pendingComments.filter((wake) => { - if (wake.id === executionWaitRequestId) return true; - if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true; - const deferredPayload = parseObject(wake.payload); - const deferredContext = parseObject( - deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], - ); - // Dedicated interaction wakes carry their own source and session - // contract. ID-only adoption must not erase that continuation. - return ( - // Durable chat work must keep its receipt, actor, source, and - // session contract through normal promotion and authorization. - !wake.idempotencyKey?.startsWith("chat-inbound:") && - !isInteractionResolutionWakePayload(deferredPayload) && - !hasInteractionContinuationWakeContext(deferredContext) && - ["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) && - queuedCommentIdsFromWakePayload(wake.payload).length > 0 - ); - }); - let adoptedCommentIds = [ - ...new Set([ - ...adoptedComments.flatMap((wake) => - queuedCommentIdsFromWakePayload(wake.payload), - ), - ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), - ]), - ]; - if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) { - adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, - agent.companyId, issueId, agentId, adoptedCommentIds); - } - await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); - const newRun = await freshIssueRun.createRun({ + durableChatReceiptId: durableRequest?.id, + allowRunCoalescing: opts.allowRunCoalescing, + executionWaitRequestId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + boardRetryReceiptId: failedChatRetry ? undefined : opts.taskRetryRequestId, + failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined, + ownsReconstructionContinuation: Boolean(ownsReconstructionContinuation), explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null, - agentId, - issueId: enrichedContextSnapshot.issueId, - invocationSource: source, - triggerDetail, - responsibleUserId: await resolveQueuedResponsibleUserId(), - wakeupRequestId: wakeupRequest.id, retryOfRunId: failedChatRetry ? durableRequest!.failedRunRetry!.failedRunId : opts.failedRunId ?? automaticParentRunId, - contextSnapshot: adoptedComments.length - ? withQueuedCommentIdsInRunContext( - enrichedContextSnapshot, - adoptedCommentIds, - adoptedComments.map((wake) => wake.id), - ) - : enrichedContextSnapshot, + contextSnapshot: enrichedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt, reconciledSourceRunId, reconciledRestoreRetryCount, + resolveResponsibleUserId: resolveQueuedResponsibleUserId, + async recordContinuationDelivery(runId) { + if (ownsReconstructionContinuation) { + await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + continuationDelivery: "delivered", continuationRunId: runId, + })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + } + if (continuationToDeliver) + await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, + runId, continuation: continuationToDeliver }); + else if (reconciledSourceRunId) + await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, + runId, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); + }, }); - - if (ownsReconstructionContinuation) { - await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ - continuationDelivery: "delivered", continuationRunId: newRun.id, - })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), - eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); - } - if (continuationToDeliver) - await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, - continuation: continuationToDeliver }); - else if (reconciledSourceRunId) - await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, - runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); - - await freshIssueRun.bindReceipts({ - wakeId: wakeupRequest.id, - runId: newRun.id, - failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined, - adopted: adoptedComments.length ? { - wakeIds: adoptedComments.map((wake) => wake.id), - payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), - } : undefined, - }); + if (!newRun) return { kind: "deferred" as const }; // executionRunId is NOT stamped here (enqueueWakeup queues the run but // doesn't start it). It will be stamped in claimQueuedRun() once the run diff --git a/server/src/services/issue-queued-comment-queue.ts b/server/src/services/issue-queued-comment-queue.ts index 87707f62942..4734dc523bc 100644 --- a/server/src/services/issue-queued-comment-queue.ts +++ b/server/src/services/issue-queued-comment-queue.ts @@ -1,8 +1,12 @@ import { createHash } from "node:crypto"; import type { IssueComment, IssueQueuedCommentQueue } from "@paperclipai/shared"; -const QUEUE_CONTEXT_KEY = "_paperclipWakeContext"; -const QUEUE_IDS_KEY = "wakeCommentIds"; +export { + queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInRunContext, + withQueuedCommentIdsInWakePayload, +} from "../lib/queued-comment-context.js"; function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -10,51 +14,6 @@ function record(value: unknown): Record { : {}; } -function uniqueIds(value: unknown): string[] { - if (!Array.isArray(value)) return []; - const seen = new Set(); - return value.flatMap((candidate) => { - if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return []; - seen.add(candidate); - return [candidate]; - }); -} - -export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] { - const payload = record(payloadValue); - const context = record(payload[QUEUE_CONTEXT_KEY]); - return uniqueIds(context[QUEUE_IDS_KEY]); -} - -export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] { - return uniqueIds(record(contextValue)[QUEUE_IDS_KEY]); -} - -export function withQueuedCommentIdsInWakePayload( - payloadValue: unknown, - ids: string[], - options: { preserveSource?: boolean } = {}, -): Record { - const payload = { ...record(payloadValue) }; - const context = { ...record(payload[QUEUE_CONTEXT_KEY]) }; - if (ids.length > 0) { - const latestId = ids[ids.length - 1]!; - context[QUEUE_IDS_KEY] = ids; - if (!options.preserveSource) { - context.wakeCommentId = latestId; - context.commentId = latestId; - payload.commentId = latestId; - } - } else { - delete context[QUEUE_IDS_KEY]; - delete context.wakeCommentId; - delete context.commentId; - delete payload.commentId; - } - payload[QUEUE_CONTEXT_KEY] = context; - return payload; -} - /** * Fingerprints one queued-comment queue: the wake id plus every comment's id * and last-updated time. A mutation that changes the queue changes this @@ -76,41 +35,6 @@ export function queuedCommentQueueRevision(input: { .slice(0, 32); } -export function withQueuedCommentIdsInRunContext( - contextValue: unknown, - ids: string[], - sourceWakeIds?: string[], -): Record { - const context = { ...record(contextValue) }; - if (sourceWakeIds) { - context.queuedCommentSourceWakeIds = uniqueIds([ - ...uniqueIds(context.queuedCommentSourceWakeIds), - ...sourceWakeIds, - ]); - } - if (ids.length > 0) { - const latestId = ids[ids.length - 1]!; - context[QUEUE_IDS_KEY] = ids; - context.wakeCommentId = latestId; - context.commentId = latestId; - } else { - delete context[QUEUE_IDS_KEY]; - delete context.wakeCommentId; - delete context.commentId; - } - - // These projections are generated immediately before dispatch. Any queue - // mutation must force them to be rebuilt from the canonical comment ids. - delete context.paperclipWake; - delete context.paperclipWakeComment; - delete context.paperclipTaskMarkdown; - delete context.paperclipTaskMarkdownCompact; - delete context.paperclipTaskMarkdownAssignment; - delete context.paperclipTaskMarkdownAssignmentCompact; - delete context.paperclipTurnContext; - return context; -} - export type QueuedCommentQueueProtocol = "paperclip_runner_v1" | "legacy"; export type QueuedCommentQueueSteeringDecision =