From eca7f8577855855ddf77690bef4f835daeba1907 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Wed, 18 Nov 2026 19:24:39 -0800 Subject: [PATCH] Admit saved task comments with the fresh issue run (#22268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip agents take work from assigned tasks and from user comments. > - A fresh task run can include saved comments when the current agent or a confirmed former assignee owns them. > - Connected chat messages and resolved interactions have separate authority and session requirements. > - Heartbeat previously decided which receipts to adopt between the three calls that saved a new run. > - One wake-queue use case now selects the receipts and commits the same run and comment lineage. ## Linked Issues or Issue Description **What existing behavior does this improve?** Assignment and comment continuation when a task needs a fresh run. **Subsystem affected** Server task admission. **Current behavior** Heartbeat chooses the comment receipts to consume, reads their source, and builds the final run context. **Proposed behavior** One application operation admits the run. Its Postgres adapter loads eligible company and agent receipts under the issue lock. The caller supplies the responsible user and records the specialized continuation. **Additional context** Builds on #22266. The existing release and queued-comment boundaries are in #13136 and #13145. ## What Changed - Keep exact resumed receipts separate from ordinary board comments. Adopt a prior assignee’s comments only when the interrupted run proves a handoff. - Use the same canonical comment ID codecs for admission and dispatch. - Test the assignment path with one or two saved comments and independent chat and interaction receipts. ## Verification On c957f8529c, 260 tests passed across the four admission, liveness and responsible-user suites. The saved Grok interrupt route passed its exactly-once continuation test. A separate Postgres run passed the ordered comment and rollback checks, both reassignment cases, and the company isolation test. Server typecheck and module boundaries passed. Check the fresh admission and handoff cases in `server/src/services/heartbeat/queue.test.ts`. Run `pnpm exec vitest run server/src/__tests__/heartbeat-comment-wake-batching.test.ts server/src/services/issue-queued-comment-queue.test.ts`, `pnpm --filter @paperclipai/server typecheck`, and `node scripts/check-module-boundaries.mjs`. ## Risks Receipt selection controls which user input reaches the next agent. The store requires the same company, issue and executor or a verified handoff. Chat and interaction receipts preserve their independent delivery. ## Model Used OpenAI GPT-6.2 Sol (`gpt-6.2-sol`) implemented the change. OpenAI Codex directed the work, reviewed the diff, and ran qualification. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/heartbeat-internals.md | 6 +- server/src/lib/queued-comment-context.ts | 88 +++++++++ .../modules/run-dispatch/adapters/postgres.ts | 2 +- .../run-dispatch/domain/wake-context.ts | 4 +- .../modules/wake-queue/adapters/postgres.ts | 45 ++++- .../wake-queue/application/fresh-issue-run.ts | 187 +++++++++++++----- .../wake-queue/application/use-cases.ts | 3 +- .../src/modules/wake-queue/domain/context.ts | 2 +- server/src/modules/wake-queue/index.ts | 4 +- server/src/services/heartbeat/queue.test.ts | 57 +++++- server/src/services/heartbeat/queue.ts | 169 ++++------------ .../services/issue-queued-comment-queue.ts | 88 +-------- 12 files changed, 372 insertions(+), 283 deletions(-) create mode 100644 server/src/lib/queued-comment-context.ts diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index ba0ce713ae4..ffe8d1dfee4 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -316,8 +316,10 @@ Keep recovery policy changes separate from retry scheduling and execution change Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`. It owns wake coalescing and batching, queued-run claims, daily heartbeat caps, concurrency and priority checks, timer admission, and native status wake intents. -The wake-queue module creates fresh task runs and binds their receipts inside -heartbeat's issue admission transaction. +The wake-queue module admits fresh task runs inside heartbeat's issue admission +transaction. It selects eligible saved comment receipts in request order and +binds them to the new run. Heartbeat supplies responsible-user resolution and +records any specialized continuation delivery before those receipts are bound. `createHeartbeatQueue` binds these operations without querying or starting work. The service supplies lifecycle effects, execution callbacks, worktree gates, and its process-wide execution and wakeup promise sets. Forwarding callbacks preserve diff --git a/server/src/lib/queued-comment-context.ts b/server/src/lib/queued-comment-context.ts new file mode 100644 index 00000000000..41148b9006e --- /dev/null +++ b/server/src/lib/queued-comment-context.ts @@ -0,0 +1,88 @@ +export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; + +function parseObject(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function uniqueIds(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const seen = new Set(); + return value.flatMap((candidate) => { + if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return []; + seen.add(candidate); + return [candidate]; + }); +} + +export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] { + const payload = parseObject(payloadValue); + const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + return uniqueIds(context[WAKE_COMMENT_IDS_KEY]); +} + +export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] { + return uniqueIds(parseObject(contextValue)[WAKE_COMMENT_IDS_KEY]); +} + +export function withQueuedCommentIdsInWakePayload( + payloadValue: unknown, + ids: string[], + options: { preserveSource?: boolean } = {}, +): Record { + const payload = { ...parseObject(payloadValue) }; + const context = { ...parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]) }; + if (ids.length > 0) { + const latestId = ids[ids.length - 1]!; + context[WAKE_COMMENT_IDS_KEY] = ids; + if (!options.preserveSource) { + context.wakeCommentId = latestId; + context.commentId = latestId; + payload.commentId = latestId; + } + } else { + delete context[WAKE_COMMENT_IDS_KEY]; + delete context.wakeCommentId; + delete context.commentId; + delete payload.commentId; + } + payload[DEFERRED_WAKE_CONTEXT_KEY] = context; + return payload; +} + +export function withQueuedCommentIdsInRunContext( + contextValue: unknown, + ids: string[], + sourceWakeIds?: string[], +): Record { + const context = { ...parseObject(contextValue) }; + if (sourceWakeIds) { + context.queuedCommentSourceWakeIds = uniqueIds([ + ...uniqueIds(context.queuedCommentSourceWakeIds), + ...sourceWakeIds, + ]); + } + if (ids.length > 0) { + const latestId = ids[ids.length - 1]!; + context[WAKE_COMMENT_IDS_KEY] = ids; + context.wakeCommentId = latestId; + context.commentId = latestId; + } else { + delete context[WAKE_COMMENT_IDS_KEY]; + delete context.wakeCommentId; + delete context.commentId; + } + + // These projections are generated immediately before dispatch. Any queue + // mutation must force them to be rebuilt from the canonical comment ids. + delete context.paperclipWake; + delete context.paperclipWakeComment; + delete context.paperclipTaskMarkdown; + delete context.paperclipTaskMarkdownCompact; + delete context.paperclipTaskMarkdownAssignment; + delete context.paperclipTaskMarkdownAssignmentCompact; + delete context.paperclipTurnContext; + return context; +} diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 592a8d1517d..810d1494915 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -5,7 +5,7 @@ import { readNativeWorkspaceSyncReference } from "../../../services/native-runti import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; import { isAgentDirectoryPreparationWait } from "../../../services/pre-dispatch-execution-evidence.js"; import { isUserCommentContinuationReason } from "../../../services/explicit-native-continuation.js"; -import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js"; +import { queuedCommentIdsFromRunContext, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../lib/queued-comment-context.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "../../../services/cancelled-workspace-preservation-notifications.js"; diff --git a/server/src/modules/run-dispatch/domain/wake-context.ts b/server/src/modules/run-dispatch/domain/wake-context.ts index 18c7716359e..26cddce4516 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.ts @@ -5,6 +5,9 @@ // independently maintained copy in either file could silently drift out of // sync with the other and change only one of the two gates that read it. +import { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js"; +export { WAKE_COMMENT_IDS_KEY } from "../../../lib/queued-comment-context.js"; + function parseObject(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -30,7 +33,6 @@ export function isPreparationWaitRetryReason(reason: string | null | undefined): } export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry"; export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry"; -export const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; export const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set([ "accepted", "answered", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index a70fa7b926a..7caa80b5f36 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -16,7 +16,7 @@ import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction- import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js"; -import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, ne, notInArray, or, sql } from "drizzle-orm"; import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; import { isGovernancePauseStop } from "../../../services/governance-pause-stop.js"; import { agentRunSupersededByFreshContinuation } from "../../../agent-run-cancellation.js"; @@ -53,13 +53,15 @@ import { executionRetryAccounting } from "../../../services/execution-recovery-a import { issueService } from "../../../services/issues.js"; import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; +import { undeliveredLegacyUserCommentIds } from "../../../services/explicit-native-continuation.js"; import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; import { - queuedCommentIdsFromWakePayload, + DEFERRED_WAKE_CONTEXT_KEY, queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, withQueuedCommentIdsInWakePayload, -} from "../../../services/issue-queued-comment-queue.js"; +} from "../../../lib/queued-comment-context.js"; import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds, readWakeChatConversationId } from "../../run-dispatch/index.js"; import { deferredWakeHoldsNoOtherConversation, otherConversationCommentIds, splitOtherConversationMessages } from "./chat-conversation-batch.js"; import { hasInteractionContinuationWakeContext } from "../domain/context.js"; @@ -85,13 +87,12 @@ import type { WakeQueueHost, WakeQueueTransaction, } from "../application/ports.js"; -import type { FreshIssueRunWriter } from "../application/fresh-issue-run.js"; +import type { FreshIssueRunStore } from "../application/fresh-issue-run.js"; import type { RunSummary } from "../application/types.js"; const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; /** Wake outcome for a deferred chat wake whose news an earlier reply already gave. */ export const ALREADY_REPORTED_WAKE_REASON = "Already reported by an earlier reply in this conversation"; -const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; @@ -1537,10 +1538,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd }; } -export function createFreshIssueRunWriter( +export function createFreshIssueRunStore( scope: TransactionScope, companyId: string, -): FreshIssueRunWriter { +): FreshIssueRunStore { const tx = requireAdmissionTx(scope, companyId); return { async saveReceipt(input) { @@ -1561,6 +1562,36 @@ export function createFreshIssueRunWriter( )).returning(); return receipt; }, + async readCommentCandidates({ issueId, agentId, handoffRunId, receiptScope, checkExecutionBlocker }) { + const [handoff] = handoffRunId ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, handoffRunId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"), + ne(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issueId)), + )) : []; + const candidates = receiptScope && (!checkExecutionBlocker || !(await getExecutionBlocker(tx, companyId, issueId))) + ? await tx.select({ + id: agentWakeupRequests.id, + reason: agentWakeupRequests.reason, + idempotencyKey: agentWakeupRequests.idempotencyKey, + payload: agentWakeupRequests.payload, + }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.agentId, handoff ? [agentId, handoff.agentId] : [agentId]), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + receiptScope.kind === "exact" ? eq(agentWakeupRequests.id, receiptScope.wakeId) : undefined, + )).orderBy(asc(agentWakeupRequests.requestedAt)) + : []; + return { handoffAgentId: handoff?.agentId ?? null, candidates }; + }, + readUndeliveredLegacyCommentIds(issueId, agentId, commentIds) { + return undeliveredLegacyUserCommentIds(tx, companyId, issueId, agentId, commentIds); + }, + notifyRunMilestones(issueId) { + return notifyChatRunMilestones(tx, { companyId, issueId }); + }, async insertQueuedRun(seed) { const [run] = await tx.insert(heartbeatRuns).values({ ...seed, companyId, status: "queued", diff --git a/server/src/modules/wake-queue/application/fresh-issue-run.ts b/server/src/modules/wake-queue/application/fresh-issue-run.ts index 52bebb8a082..a5c7da4a4f3 100644 --- a/server/src/modules/wake-queue/application/fresh-issue-run.ts +++ b/server/src/modules/wake-queue/application/fresh-issue-run.ts @@ -1,5 +1,14 @@ +import { isUuidLike } from "@paperclipai/shared"; +import { + DEFERRED_WAKE_CONTEXT_KEY, + queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInRunContext, + withQueuedCommentIdsInWakePayload, +} from "../../../lib/queued-comment-context.js"; +import { hasInteractionContinuationWakeContext, isInteractionResolutionWakePayload } from "../domain/context.js"; +import { parseObject, readNonEmptyString } from "../domain/values.js"; import type { DurableWakeAdmissionReceipt } from "./ports.js"; -import { readNonEmptyString } from "../domain/values.js"; type WakeReceiptSeed = { agentId: string; @@ -33,8 +42,24 @@ type SaveWakeReceipt = | { kind: "insert"; seed: WakeReceiptSeed & Partial } | { kind: "promote"; wakeId: string; agentId: string; payload: WakeReceiptSeed["payload"] }; -export type FreshIssueRunWriter = { +type DeferredCommentCandidate = { + id: string; + reason: string | null; + idempotencyKey: string | null; + payload: unknown; +}; + +export type FreshIssueRunStore = { saveReceipt(input: SaveWakeReceipt): Promise<{ id: string } | undefined>; + readCommentCandidates(input: { + issueId: string; + agentId: string; + handoffRunId: string | null; + receiptScope: { kind: "all" } | { kind: "exact"; wakeId: string } | null; + checkExecutionBlocker: boolean; + }): Promise<{ handoffAgentId: string | null; candidates: DeferredCommentCandidate[] }>; + readUndeliveredLegacyCommentIds(issueId: string, agentId: string, commentIds: string[]): Promise; + notifyRunMilestones(issueId: string): Promise; insertQueuedRun(seed: QueuedRunSeed): Promise; bindReceipts(input: { wakeId: string; @@ -44,53 +69,115 @@ export type FreshIssueRunWriter = { }): Promise; }; -/** - * Runs under heartbeat's existing issue lock. The caller validates admission, - * selects saved comments and records specialized continuation delivery between - * run creation and receipt binding. - */ -export function createFreshIssueRunAdmission(writer: FreshIssueRunWriter) { - return { - createReceipt(input: WakeReceiptSeed & { - durableReceipt?: DurableWakeAdmissionReceipt; - deferredChatReceiptId?: string; - }) { - const { durableReceipt, deferredChatReceiptId, ...seed } = input; - // A deferred durable chat wake keeps its original identity and source. - return writer.saveReceipt(deferredChatReceiptId - ? { kind: "promote", wakeId: deferredChatReceiptId, agentId: seed.agentId, payload: seed.payload } - : { - kind: "insert", - seed: durableReceipt - ? { ...seed, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } - : seed, - }); - }, - createRun(input: Omit & { - explicitContinuationRunId: string | null; - reconciledSourceRunId: string | null; - reconciledRestoreRetryCount: number | null; - issueId: unknown; - }) { - const { - explicitContinuationRunId, reconciledSourceRunId, reconciledRestoreRetryCount, issueId, ...seed - } = input; - const scopedIssueId = readNonEmptyString(issueId); - return writer.insertQueuedRun({ - ...seed, - ...(explicitContinuationRunId ? { id: explicitContinuationRunId } : {}), - scopeKind: scopedIssueId ? "issue" : "company", - issueId: scopedIssueId, - retryOfRunId: reconciledSourceRunId || seed.retryOfRunId, - sessionIdBefore: explicitContinuationRunId ? null : seed.sessionIdBefore, - ...(reconciledRestoreRetryCount !== null ? { - scheduledRetryAttempt: reconciledRestoreRetryCount, - scheduledRetryReason: "transient_failure", - } : {}), - }); - }, - bindReceipts: writer.bindReceipts, +type FreshIssueRunInput = { + receipt: WakeReceiptSeed; + issue: { id: string; assigneeAgentId: string | null; isConversation: boolean }; + durableReceipt?: DurableWakeAdmissionReceipt; + durableChatReceiptId?: string; + allowRunCoalescing?: boolean; + executionWaitRequestId?: string; + queuedCommentInterruptId?: string; + queuedCommentRequestId?: string; + boardRetryReceiptId?: string; + failedChatRetryReceiptId?: string; + ownsReconstructionContinuation: boolean; + contextSnapshot: Record; + sessionIdBefore: string | null; + continuationAttempt: number; + explicitContinuationRunId: string | null; + retryOfRunId: string | null; + reconciledSourceRunId: string | null; + reconciledRestoreRetryCount: number | null; + resolveResponsibleUserId(): Promise; + recordContinuationDelivery(runId: string): Promise; +}; + +function isOrdinaryBoardComment(wake: DeferredCommentCandidate): boolean { + const payload = parseObject(wake.payload); + const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + // Durable chat and interaction wakes carry their own actor, source, and session. + return !wake.idempotencyKey?.startsWith("chat-inbound:") && + !isInteractionResolutionWakePayload(payload) && + !hasInteractionContinuationWakeContext(context) && + ["issue_commented", "issue_reopened_via_comment"].includes(String(context.wakeReason ?? wake.reason)) && + queuedCommentIdsFromWakePayload(payload).length > 0; +} + +/** Admit one new run and its saved inputs using the caller's existing issue lock. */ +export function createFreshIssueRunAdmission(store: FreshIssueRunStore) { + return async function admitFreshIssueRun(input: FreshIssueRunInput): Promise { + const { receipt, durableReceipt, durableChatReceiptId, executionWaitRequestId, issue, contextSnapshot } = input; + const { agentId, source, triggerDetail, reason, payload } = receipt; + + // A deferred durable chat wake keeps its original identity and source. + const primary = await store.saveReceipt(durableChatReceiptId && durableChatReceiptId === executionWaitRequestId + ? { kind: "promote", wakeId: durableChatReceiptId, agentId, payload } + : { + kind: "insert", + seed: durableReceipt ? { ...receipt, id: durableReceipt.id, requestedAt: durableReceipt.requestedAt } : receipt, + }); + if (!primary) return undefined; + + // Only a verified reassignment can bring a previous executor's saved comments. + const interruptedRunId = readNonEmptyString(contextSnapshot.interruptedRunId); + const handoffRunId = source === "assignment" && reason === "issue_assigned" && + issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId) + ? interruptedRunId : null; + + // Chat keeps separate batches, but an exact resumed receipt must be consumed with its successor. + const canCoalesceComments = !durableChatReceiptId && !issue.isConversation && input.allowRunCoalescing !== false; + const { handoffAgentId, candidates } = await store.readCommentCandidates({ + issueId: issue.id, + agentId, + handoffRunId, + receiptScope: canCoalesceComments || (executionWaitRequestId && input.boardRetryReceiptId) + ? { kind: "all" } + : executionWaitRequestId ? { kind: "exact", wakeId: executionWaitRequestId } : null, + checkExecutionBlocker: !input.ownsReconstructionContinuation, + }); + const adopted = candidates.filter((wake) => wake.id === executionWaitRequestId || + wake.id === input.queuedCommentInterruptId || wake.id === input.queuedCommentRequestId || + isOrdinaryBoardComment(wake)); + const adoptedWakeIds = adopted.map((wake) => wake.id); + let commentIds = [...new Set([ + ...adopted.flatMap((wake) => queuedCommentIdsFromWakePayload(wake.payload)), + ...queuedCommentIdsFromRunContext(contextSnapshot), + ])]; + if (input.queuedCommentRequestId || input.boardRetryReceiptId || handoffAgentId) { + commentIds = await store.readUndeliveredLegacyCommentIds(issue.id, agentId, commentIds); + } + + await store.notifyRunMilestones(issue.id); + const responsibleUserId = await input.resolveResponsibleUserId(); + const scopedIssueId = readNonEmptyString(contextSnapshot.issueId); + const run = await store.insertQueuedRun({ + ...(input.explicitContinuationRunId ? { id: input.explicitContinuationRunId } : {}), + agentId, + scopeKind: scopedIssueId ? "issue" : "company", + issueId: scopedIssueId, + invocationSource: source, + triggerDetail, + responsibleUserId, + wakeupRequestId: primary.id, + retryOfRunId: input.reconciledSourceRunId || input.retryOfRunId, + contextSnapshot: adopted.length ? withQueuedCommentIdsInRunContext(contextSnapshot, commentIds, adoptedWakeIds) : contextSnapshot, + sessionIdBefore: input.explicitContinuationRunId ? null : input.sessionIdBefore, + continuationAttempt: input.continuationAttempt, + ...(input.reconciledRestoreRetryCount !== null ? { + scheduledRetryAttempt: input.reconciledRestoreRetryCount, + scheduledRetryReason: "transient_failure", + } : {}), + }); + await input.recordContinuationDelivery(run.id); + await store.bindReceipts({ + wakeId: primary.id, + runId: run.id, + failedChatRetryReceiptId: input.failedChatRetryReceiptId, + adopted: adopted.length ? { + wakeIds: adoptedWakeIds, + payload: withQueuedCommentIdsInWakePayload(payload, commentIds), + } : undefined, + }); + return run; }; } diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts index fb6a31c0f61..ccd0817f78b 100644 --- a/server/src/modules/wake-queue/application/use-cases.ts +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -1,4 +1,5 @@ import { readWakeChatConversationId } from "../../run-dispatch/index.js"; +import { DEFERRED_WAKE_CONTEXT_KEY } from "../../../lib/queued-comment-context.js"; import { enrichPromotedWakeContext, wakeMustNotJoinActiveRun, withMergedWakeReasons } from "../domain/context.js"; import { QUEUED_MESSAGES_DELIVERED_REASON } from "../domain/policy.js"; import { @@ -36,8 +37,6 @@ import type { import type { PostCommitEffect, ReleaseOutcome } from "./types.js"; import { WakeQueueApplicationError } from "./types.js"; -const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; - const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair"; const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([ "execution_review_requested", diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts index 51a7c6e91f7..20607eba742 100644 --- a/server/src/modules/wake-queue/domain/context.ts +++ b/server/src/modules/wake-queue/domain/context.ts @@ -114,7 +114,7 @@ export function hasRetainedExternalChatRetry(context: unknown): boolean { (activeSource === "issue.interaction.respond" && active.externalChatContinuation === true)); } -function isInteractionResolutionWakePayload(payload: Record | null | undefined): boolean { +export function isInteractionResolutionWakePayload(payload: Record | null | undefined): boolean { return readNonEmptyString(payload?.mutation) === "interaction"; } diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 33fd109e102..2ad2b1247ab 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -1,7 +1,7 @@ import type { Db } from "@paperclipai/db"; import { createAdmissionTransactionScope as buildAdmissionTransactionScope, - createFreshIssueRunWriter, + createFreshIssueRunStore, createPostgresWakeQueueAdapter, createWakeAdmissionReader, createWakeAdmissionWriter, @@ -114,7 +114,7 @@ export type WakeQueue = ReturnType; /** Uses the transaction that already owns the issue's admission lock. */ export function bindFreshIssueRunAdmission(companyId: string, tx: Db) { - return createFreshIssueRunAdmission(createFreshIssueRunWriter(buildAdmissionTransactionScope(companyId, tx), companyId)); + return createFreshIssueRunAdmission(createFreshIssueRunStore(buildAdmissionTransactionScope(companyId, tx), companyId)); } /** diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index a67dd4cc4a9..8808e4871e9 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -7,11 +7,11 @@ import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; -import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js"; +import { createFreshIssueRunStore, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; -import { bindFreshIssueRunAdmission, createWakeQueue } from "../../modules/wake-queue/index.js"; +import { createWakeQueue } from "../../modules/wake-queue/index.js"; import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; @@ -494,6 +494,49 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { company, agent, issue, wake, run }; } + it.each([true, false])("adopts only eligible saved comments from the exact reassignment source (%s)", async (matchingIssue) => { + const { user, company, agent, issue, deps, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, + conversationState: null, status: "todo" }).where(eq(issues.id, issue.id)); + const [previousAgent] = await db.insert(agents).values({ companyId: company.id, + name: "Previous assignee", adapterType: "process" }).returning(); + const [interruptedRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: previousAgent.id, + status: "cancelled", errorCode: "issue_reassigned", finishedAt: new Date(), + contextSnapshot: { issueId: matchingIssue ? issue.id : randomUUID() } }).returning(); + const [earlier, later] = await db.insert(issueComments).values(["Earlier", "Later"].map(body => ({ + companyId: company.id, issueId: issue.id, authorType: "user" as const, authorUserId: user.id, body, + }))).returning(); + const [local, handoff, chat, interaction] = await db.insert(agentWakeupRequests).values([ + { agentId: agent.id, commentId: later.id, requestedAt: new Date(2) }, + { agentId: previousAgent.id, commentId: earlier.id, requestedAt: new Date(1) }, + { agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(3), idempotencyKey: `chat-inbound:${randomUUID()}` }, + { agentId: agent.id, commentId: randomUUID(), requestedAt: new Date(4), mutation: "interaction" }, + ].map(({ agentId, commentId, requestedAt, idempotencyKey, mutation }) => ({ + companyId: company.id, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution", + requestedAt, idempotencyKey, requestedByActorType: "user", requestedByActorId: user.id, + payload: withQueuedCommentIdsInWakePayload({ issueId: issue.id, mutation }, [commentId]), + }))).returning(); + + const run = await queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "issue_assigned", requestedByActorType: "system", + payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, interruptedRunId: interruptedRun.id }, + }); + + const commentIds = matchingIssue ? [earlier.id, later.id] : [later.id]; + expect(run).toMatchObject({ status: "queued", responsibleUserId: user.id, invocationSource: "assignment", + contextSnapshot: { wakeCommentIds: commentIds, wakeCommentId: later.id, + queuedCommentSourceWakeIds: matchingIssue ? [handoff.id, local.id] : [local.id] } }); + const receipts = new Map((await db.select().from(agentWakeupRequests)).map(wake => [wake.id, wake])); + expect(receipts.get(run!.wakeupRequestId!)).toMatchObject({ runId: run!.id, source: "assignment", + requestedByActorType: "system", payload: { _paperclipWakeContext: { wakeCommentIds: commentIds } } }); + expect(receipts.get(local.id)).toMatchObject({ status: "coalesced", runId: run!.id, requestedByActorId: user.id }); + expect(receipts.get(handoff.id)).toMatchObject(matchingIssue + ? { status: "coalesced", runId: run!.id, agentId: previousAgent.id } + : { status: "deferred_issue_execution", runId: null }); + for (const receipt of [chat, interaction]) expect(receipts.get(receipt.id)).toEqual(receipt); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("only promotes a deferred chat receipt for its agent and binds receipts within its company", async () => { const local = await fixture("queued"); const [foreignCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); @@ -510,13 +553,11 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { }).returning(); await db.transaction(async (tx) => { - const admission = bindFreshIssueRunAdmission(local.company.id, tx as unknown as Db); - expect(await admission.createReceipt({ - deferredChatReceiptId: localDeferredWake.id, agentId: foreignAgent.id, source: "automation", - triggerDetail: null, reason: null, payload: null, requestedByActorType: null, - requestedByActorId: null, idempotencyKey: null, + const store = createFreshIssueRunStore(TransactionScope.create(local.company.id, tx), local.company.id); + expect(await store.saveReceipt({ + kind: "promote", wakeId: localDeferredWake.id, agentId: foreignAgent.id, payload: null, })).toBeUndefined(); - await admission.bindReceipts({ + await store.bindReceipts({ wakeId: foreignWake.id, runId: local.run.id, failedChatRetryReceiptId: foreignWake.id, adopted: { wakeIds: [foreignWake.id, localDeferredWake.id], payload: { replaced: true } }, }); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index fb0a78e4c5c..ceeb8afa2e7 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -144,11 +144,12 @@ import { isUniqueViolation } from "../../db-errors.js"; import { publishLiveEvent } from "../live-events.js"; import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js"; import { + DEFERRED_WAKE_CONTEXT_KEY, queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext, -} from "../issue-queued-comment-queue.js"; +} from "../../lib/queued-comment-context.js"; import { parseObject } from "../../adapters/utils.js"; import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; @@ -278,7 +279,7 @@ export interface HeartbeatQueueDependencies extends Pick>'issueId' = ${issue.id}`, - or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), - )).then(rows => rows[0] ?? null) - : null; - // External input owns an exact authorized source batch. Board input - // must retain its separate receipt, including during normal resume. - const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false; - // A resumed receipt already passed admission under this issue lock. - // Consume it with its successor even when chat keeps other messages - // in separate turns, or finalization will deliver it a second time. - const pendingComments = - (executionWaitRequestId || canCoalesceComments) && - (ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id))) - ? await tx - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, - // Chat Retry already pins its complete authorized source - // batch. Keep separate board input for its own admission. - canCoalesceComments || (opts.taskRetryRequestId && !failedChatRetry) - ? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!), - ), - ) - .orderBy(asc(agentWakeupRequests.requestedAt)) - : []; - const adoptedComments = pendingComments.filter((wake) => { - if (wake.id === executionWaitRequestId) return true; - if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true; - const deferredPayload = parseObject(wake.payload); - const deferredContext = parseObject( - deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], - ); - // Dedicated interaction wakes carry their own source and session - // contract. ID-only adoption must not erase that continuation. - return ( - // Durable chat work must keep its receipt, actor, source, and - // session contract through normal promotion and authorization. - !wake.idempotencyKey?.startsWith("chat-inbound:") && - !isInteractionResolutionWakePayload(deferredPayload) && - !hasInteractionContinuationWakeContext(deferredContext) && - ["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) && - queuedCommentIdsFromWakePayload(wake.payload).length > 0 - ); - }); - let adoptedCommentIds = [ - ...new Set([ - ...adoptedComments.flatMap((wake) => - queuedCommentIdsFromWakePayload(wake.payload), - ), - ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), - ]), - ]; - if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) { - adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, - agent.companyId, issueId, agentId, adoptedCommentIds); - } - await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); - const newRun = await freshIssueRun.createRun({ + durableChatReceiptId: durableRequest?.id, + allowRunCoalescing: opts.allowRunCoalescing, + executionWaitRequestId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + boardRetryReceiptId: failedChatRetry ? undefined : opts.taskRetryRequestId, + failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined, + ownsReconstructionContinuation: Boolean(ownsReconstructionContinuation), explicitContinuationRunId: explicitContinuation ? explicitContinuationRunId : null, - agentId, - issueId: enrichedContextSnapshot.issueId, - invocationSource: source, - triggerDetail, - responsibleUserId: await resolveQueuedResponsibleUserId(), - wakeupRequestId: wakeupRequest.id, retryOfRunId: failedChatRetry ? durableRequest!.failedRunRetry!.failedRunId : opts.failedRunId ?? automaticParentRunId, - contextSnapshot: adoptedComments.length - ? withQueuedCommentIdsInRunContext( - enrichedContextSnapshot, - adoptedCommentIds, - adoptedComments.map((wake) => wake.id), - ) - : enrichedContextSnapshot, + contextSnapshot: enrichedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt, reconciledSourceRunId, reconciledRestoreRetryCount, + resolveResponsibleUserId: resolveQueuedResponsibleUserId, + async recordContinuationDelivery(runId) { + if (ownsReconstructionContinuation) { + await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + continuationDelivery: "delivered", continuationRunId: runId, + })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + } + if (continuationToDeliver) + await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, + runId, continuation: continuationToDeliver }); + else if (reconciledSourceRunId) + await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, + runId, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); + }, }); - - if (ownsReconstructionContinuation) { - await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ - continuationDelivery: "delivered", continuationRunId: newRun.id, - })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), - eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); - } - if (continuationToDeliver) - await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, - continuation: continuationToDeliver }); - else if (reconciledSourceRunId) - await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, - runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); - - await freshIssueRun.bindReceipts({ - wakeId: wakeupRequest.id, - runId: newRun.id, - failedChatRetryReceiptId: failedChatRetry ? opts.taskRetryRequestId : undefined, - adopted: adoptedComments.length ? { - wakeIds: adoptedComments.map((wake) => wake.id), - payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), - } : undefined, - }); + if (!newRun) return { kind: "deferred" as const }; // executionRunId is NOT stamped here (enqueueWakeup queues the run but // doesn't start it). It will be stamped in claimQueuedRun() once the run diff --git a/server/src/services/issue-queued-comment-queue.ts b/server/src/services/issue-queued-comment-queue.ts index 87707f62942..4734dc523bc 100644 --- a/server/src/services/issue-queued-comment-queue.ts +++ b/server/src/services/issue-queued-comment-queue.ts @@ -1,8 +1,12 @@ import { createHash } from "node:crypto"; import type { IssueComment, IssueQueuedCommentQueue } from "@paperclipai/shared"; -const QUEUE_CONTEXT_KEY = "_paperclipWakeContext"; -const QUEUE_IDS_KEY = "wakeCommentIds"; +export { + queuedCommentIdsFromRunContext, + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInRunContext, + withQueuedCommentIdsInWakePayload, +} from "../lib/queued-comment-context.js"; function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -10,51 +14,6 @@ function record(value: unknown): Record { : {}; } -function uniqueIds(value: unknown): string[] { - if (!Array.isArray(value)) return []; - const seen = new Set(); - return value.flatMap((candidate) => { - if (typeof candidate !== "string" || !candidate || seen.has(candidate)) return []; - seen.add(candidate); - return [candidate]; - }); -} - -export function queuedCommentIdsFromWakePayload(payloadValue: unknown): string[] { - const payload = record(payloadValue); - const context = record(payload[QUEUE_CONTEXT_KEY]); - return uniqueIds(context[QUEUE_IDS_KEY]); -} - -export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] { - return uniqueIds(record(contextValue)[QUEUE_IDS_KEY]); -} - -export function withQueuedCommentIdsInWakePayload( - payloadValue: unknown, - ids: string[], - options: { preserveSource?: boolean } = {}, -): Record { - const payload = { ...record(payloadValue) }; - const context = { ...record(payload[QUEUE_CONTEXT_KEY]) }; - if (ids.length > 0) { - const latestId = ids[ids.length - 1]!; - context[QUEUE_IDS_KEY] = ids; - if (!options.preserveSource) { - context.wakeCommentId = latestId; - context.commentId = latestId; - payload.commentId = latestId; - } - } else { - delete context[QUEUE_IDS_KEY]; - delete context.wakeCommentId; - delete context.commentId; - delete payload.commentId; - } - payload[QUEUE_CONTEXT_KEY] = context; - return payload; -} - /** * Fingerprints one queued-comment queue: the wake id plus every comment's id * and last-updated time. A mutation that changes the queue changes this @@ -76,41 +35,6 @@ export function queuedCommentQueueRevision(input: { .slice(0, 32); } -export function withQueuedCommentIdsInRunContext( - contextValue: unknown, - ids: string[], - sourceWakeIds?: string[], -): Record { - const context = { ...record(contextValue) }; - if (sourceWakeIds) { - context.queuedCommentSourceWakeIds = uniqueIds([ - ...uniqueIds(context.queuedCommentSourceWakeIds), - ...sourceWakeIds, - ]); - } - if (ids.length > 0) { - const latestId = ids[ids.length - 1]!; - context[QUEUE_IDS_KEY] = ids; - context.wakeCommentId = latestId; - context.commentId = latestId; - } else { - delete context[QUEUE_IDS_KEY]; - delete context.wakeCommentId; - delete context.commentId; - } - - // These projections are generated immediately before dispatch. Any queue - // mutation must force them to be rebuilt from the canonical comment ids. - delete context.paperclipWake; - delete context.paperclipWakeComment; - delete context.paperclipTaskMarkdown; - delete context.paperclipTaskMarkdownCompact; - delete context.paperclipTaskMarkdownAssignment; - delete context.paperclipTaskMarkdownAssignmentCompact; - delete context.paperclipTurnContext; - return context; -} - export type QueuedCommentQueueProtocol = "paperclip_runner_v1" | "legacy"; export type QueuedCommentQueueSteeringDecision =