diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index ffe8d1dfee4..ee6b4b7b282 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -313,13 +313,16 @@ and shared execution sets so separate service instances keep the same ownership and shutdown barriers. Cleanup single-flight state stays at module scope. Keep recovery policy changes separate from retry scheduling and execution changes. -Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`. -It owns wake coalescing and batching, queued-run claims, daily heartbeat caps, -concurrency and priority checks, timer admission, and native status wake intents. -The wake-queue module admits fresh task runs inside heartbeat's issue admission -transaction. It selects eligible saved comment receipts in request order and -binds them to the new run. Heartbeat supplies responsible-user resolution and -records any specialized continuation delivery before those receipts are bound. +Heartbeat coordinates wakeup dispatch, queued-run claims, daily heartbeat caps, +concurrency and priority checks, timer admission, and native status wake intents +in `server/src/services/heartbeat/queue.ts`. The wake-queue module owns the +company and issue lock, the admission transaction, and the handoff after commit +for both fresh task runs and wakes arriving behind an active execution. Heartbeat +uses the locked transaction for its specialized eligibility gates and continuation +writes. Wake-queue selects eligible saved comment receipts in request order and +binds them to the new run. Chat authorization and its admission callback run in +the same transaction. Authorization contention defers the wake and rolls back +without publishing buffered effects. `createHeartbeatQueue` binds these operations without querying or starting work. The service supplies lifecycle effects, execution callbacks, worktree gates, and its process-wide execution and wakeup promise sets. Forwarding callbacks preserve diff --git a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts index f812d312305..5ae75ddcdcb 100644 --- a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts +++ b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts @@ -86,8 +86,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, }); - callbacks.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); - callbacks.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + callbacks.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake); const queue = createHeartbeatQueue(db, callbacks); const cancel = createStalledQueuedRunCancel(callbacks); const sweep = (beforeCancel?: () => Promise) => assignedWorkSweep(db, { diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 268f13836f5..918e6802c09 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -16,7 +16,7 @@ import { settleUnrecoverableExecutions } from "../services/execution-recovery-re import { recordRunPresentationDecision } from "../services/heartbeat/run-completion.js"; import { environmentService } from "../services/environments.js"; import { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } from "../services/workspace-runtime.js"; -import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; import { chatFailedRetryNoticeWork } from "../services/chat-failed-retry-notice-work.js"; import { issueTreeControlService } from "../services/issue-tree-control.js"; import { chatInboundWakeNoticeWork } from "../services/chat-inbound-wake-notice-work.js"; @@ -69831,8 +69831,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } else if (mode === "requester merge") { await db.transaction(async tx => { await tx.execute(sql`select id from issues where id = ${context.issue.id} for update`); - await createWakeAdmissionWriter().mergeIntoExistingDeferredWake( - createAdmissionTransactionScope(context.fixture.companyId, tx as unknown as Db), { + await createWakeAdmissionWriter(tx, context.fixture.companyId).mergeIntoExistingDeferredWake({ companyId: context.fixture.companyId, existingDeferredWakeId: originalReceipt.id, mergedPayload: originalReceipt.payload ?? {}, nextCoalescedCount: 1, manualUserWakeActorId: originalReceipt.requestedByActorId!, diff --git a/server/src/__tests__/chat-run-milestone-work.test.ts b/server/src/__tests__/chat-run-milestone-work.test.ts index 1fec371eed3..03edf1a6401 100644 --- a/server/src/__tests__/chat-run-milestone-work.test.ts +++ b/server/src/__tests__/chat-run-milestone-work.test.ts @@ -14,7 +14,7 @@ import { questionResponseDeliveryService } from "../services/question-response-d import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js"; import { createDiscardQueuedComment, createReorderQueuedComments } from "../modules/wake-queue/application/queued-comment-use-cases.js"; import { createRunDispatch } from "../modules/run-dispatch/index.js"; -import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; @@ -335,8 +335,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const seeded = await run(f, { contextSnapshot: { issueId: f.issueId, source: "issue_assigned" } }); await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); await db.transaction(async tx => { - await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( - createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { + await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({ companyId: f.companyId, activeExecutionRunId: seeded.id, activeExecutionRunStatus: seeded.status, mergedContextSnapshot: chatContextSnapshot(), agentId: f.agentId, source: "on_demand", triggerDetail: null, payload: { issueId: f.issueId }, requestedByActorType: "user", requestedByActorId: f.userId, idempotencyKey: null, diff --git a/server/src/__tests__/helpers/heartbeat-queue.ts b/server/src/__tests__/helpers/heartbeat-queue.ts index dc3496919c7..641a643c609 100644 --- a/server/src/__tests__/helpers/heartbeat-queue.ts +++ b/server/src/__tests__/helpers/heartbeat-queue.ts @@ -3,10 +3,11 @@ import { heartbeatRuns } from "@paperclipai/db"; import { and, eq, inArray } from "drizzle-orm"; import { vi, type Mock, type Mocked } from "vitest"; import { createRunDispatch } from "../../modules/run-dispatch/index.js"; -import type { createWakeQueue } from "../../modules/wake-queue/index.js"; +import { createWakeQueue } from "../../modules/wake-queue/index.js"; +import { filterZombieCoalesceTarget } from "../../services/heartbeat.js"; import { createHeartbeatRunState } from "../../services/heartbeat/run-state.js"; -import { createHeartbeatRunPreparation } from "../../services/heartbeat/run-preparation.js"; -import type { HeartbeatQueueDependencies } from "../../services/heartbeat/queue.js"; +import { createHeartbeatRunPreparation, mergeCoalescedContextSnapshot } from "../../services/heartbeat/run-preparation.js"; +import { shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "../../services/heartbeat/queue.js"; type Run = typeof heartbeatRuns.$inferSelect; type MockedCallback = "getAgent" | "setRunStatusFromLive" | "setWakeupStatus" | "appendRunEvent" | "finalizeAgentStatus" | "getAgentInvokability" | "getWorktreeExecutionCutoff" | "applyRunDispatchPostCommitEffects" | "getSchedulingSuppression" | "canStartNewExternalBotWork" | "executeRun" | "applyWakeQueuePostCommitEffects" | "releaseIssueExecutionAndPromote" | "publishRunLifecyclePluginEvent" | "cancelRunInternal" | "cancelActiveForAgentInternal" | "resumeExecutionWaitComments" | "resumeQueuedCommentInterrupt" | "resumeSavedLegacyComments" | "sweepPendingCleanupLeases"; @@ -22,6 +23,14 @@ type QueueTestDependencies = HeartbeatQueueDependencies & { [K in MockedCallback export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependencies { const state = createHeartbeatRunState(db); const preparation = createHeartbeatRunPreparation(db); + const admission = createWakeQueue(db, { + resolveResponsibleUserId: async () => { throw new Error("Unexpected wake promotion"); }, + getRoutineEnv: async () => { throw new Error("Unexpected wake promotion"); }, + resolveSessionBeforeForWakeup: async () => null, + wakeAdmissionHelpers: { filterZombieCoalesceTarget, mergeCoalescedContextSnapshot, + shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); return { ...preparation, ...state, @@ -53,8 +62,7 @@ export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependenc instanceSettings: { getExperimental: vi.fn() }, runDispatch: createRunDispatch(db), wakeQueue: { - createAdmissionTransactionScope: vi.fn["createAdmissionTransactionScope"]>(), - admitWakeBehindIssueExecution: vi.fn["admitWakeBehindIssueExecution"]>(), + admitIssueWake: vi.fn(admission.admitIssueWake), }, activeRunExecutions: new Set(), activeRunExecutionPromises: new Set>(), diff --git a/server/src/__tests__/slack-conversation-lifecycle.test.ts b/server/src/__tests__/slack-conversation-lifecycle.test.ts index 6e2f5bf7031..af27967391f 100644 --- a/server/src/__tests__/slack-conversation-lifecycle.test.ts +++ b/server/src/__tests__/slack-conversation-lifecycle.test.ts @@ -12,7 +12,7 @@ import { createChatPublicationResolution } from "../services/chat-channels/publi import { createChatPublicationTaskControl } from "../services/chat-channels/publications/task-control.js"; import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; -import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; import { applyRunnerGoalPrpEvent } from "../services/runner-goals.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { setTimeout as delay } from "node:timers/promises"; @@ -205,7 +205,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const key = `issue-monitor-recovery:${f.issueId}:${issue!.monitorNextCheckAt!.toISOString()}`; await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId)); await db.transaction(async tx => { - await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { + await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({ companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { issueId: f.issueId }, source: "automation", triggerDetail: "system", payload: { issueId: f.issueId }, requestedByActorType: "system", requestedByActorId: null, diff --git a/server/src/modules/wake-queue/adapters/chat-wake-conversation-batch.test.ts b/server/src/modules/wake-queue/adapters/chat-wake-conversation-batch.test.ts index 760708c4549..6c5b3f51489 100644 --- a/server/src/modules/wake-queue/adapters/chat-wake-conversation-batch.test.ts +++ b/server/src/modules/wake-queue/adapters/chat-wake-conversation-batch.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, asc, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { agentWakeupRequests, chatConversations, chatDeliveries, chatMessageLinks, createDb, heartbeatRuns, - issueComments, issues, type Db } from "@paperclipai/db"; + issueComments, issues } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; import { seedBoundChatAuthorization } from "../../../__tests__/helpers/bound-chat-authorization.js"; import { authorizeChatConversationForBoundRun } from "../../../services/native-runtime/chat-attachment-reuse.js"; @@ -11,7 +11,7 @@ import { saveUndeliveredUserComments } from "../../../services/work-intent-outbo import { queuedCommentIdsFromWakePayload } from "../../../services/issue-queued-comment-queue.js"; import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "../application/use-cases.js"; import type { WakeAdmissionActiveExecutionRun } from "../application/ports.js"; -import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionReader, +import { createPostgresWakeQueueAdapter, createWakeAdmissionReader, createWakeAdmissionWriter } from "./postgres.js"; const support = await getEmbeddedPostgresTestSupport(); @@ -31,17 +31,6 @@ describe.skipIf(!support.supported)("chat wake batches stay within one conversat let otherConversationId: string; let deferOnly = true; - const admit = createAdmitWakeBehindIssueExecution({ - reader: createWakeAdmissionReader(), - writer: createWakeAdmissionWriter(), - helpers: { - filterZombieCoalesceTarget: (target) => target, - mergeCoalescedContextSnapshot, - shouldDeferFollowupWakeForSameIssue: () => deferOnly, - shouldQueueFollowupForRunningIssueWake: () => false, - }, - }); - beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("wake-chat-conversation-batch-"); db = createDb(database.connectionString); @@ -102,7 +91,16 @@ describe.skipIf(!support.supported)("chat wake batches stay within one conversat async function admitWake(contextSnapshot: Record, activeExecutionRun?: WakeAdmissionActiveExecutionRun) { const commentId = contextSnapshot.wakeCommentId as string; const chat = contextSnapshot.source === "chat:slack"; - return db.transaction(async (tx) => admit(createAdmissionTransactionScope(f.companyId, tx as unknown as Db), { + return db.transaction((tx) => createAdmitWakeBehindIssueExecution({ + reader: createWakeAdmissionReader(tx, f.companyId), + writer: createWakeAdmissionWriter(tx, f.companyId), + helpers: { + filterZombieCoalesceTarget: (target) => target, + mergeCoalescedContextSnapshot, + shouldDeferFollowupWakeForSameIssue: () => deferOnly, + shouldQueueFollowupForRunningIssueWake: () => false, + }, + })({ companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, agentNameKey: "chat runner", issueExecutionAgentNameKey: null, activeExecutionRun: activeExecutionRun ?? { id: f.runId, agentId: f.agentId, status: "running", diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 09976ee2e64..81ddbb510c8 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -26,13 +26,11 @@ import { startEmbeddedPostgresTestDatabase, } from "../../../__tests__/helpers/embedded-postgres.js"; import { - createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter, } from "./postgres.js"; import type { WakeQueuePostgresAdapterDeps } from "./postgres.js"; import { createReleaseIssueExecution } from "../application/use-cases.js"; -import type { TransactionScope } from "../application/ports.js"; import { withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js"; // Proves the atomicity and company-scope properties the security review @@ -1426,8 +1424,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }).returning(); await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, runId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); - const operation = db.transaction(async tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( - createAdmissionTransactionScope(companyId, tx as unknown as Db), { + const operation = db.transaction(async tx => createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({ companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "queued", mergedContextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id, newComment.id]), source: "on_demand", triggerDetail: "manual", payload: { issueId, commentId: newComment.id }, @@ -1463,15 +1460,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, companyId)) .orderBy(agentWakeupRequests.id); - const writer = createWakeAdmissionWriter(); await expect( db.transaction(async (tx) => { - const scope = createAdmissionTransactionScope( - companyId, - tx as unknown as Db, - ); - await writer.mergeIntoExistingDeferredWake(scope, { + await createWakeAdmissionWriter(tx, companyId).mergeIntoExistingDeferredWake({ companyId, existingDeferredWakeId: wakeId, mergedPayload: { issueId, changedByMerge: true }, @@ -1512,12 +1504,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { const agentId = await seedAgent({ companyId }); const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); const wakeId = await seedDeferredWake({ companyId, agentId, issueId }); - const writer = createWakeAdmissionWriter(); await expect( db.transaction(async (tx) => { - const scope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db); - await writer.mergeIntoExistingDeferredWake(scope, { + await createWakeAdmissionWriter(tx, otherCompanyId).mergeIntoExistingDeferredWake({ companyId: otherCompanyId, existingDeferredWakeId: wakeId, mergedPayload: { issueId, foo: "bar" }, @@ -1543,7 +1533,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }); try { await db.transaction(async tx => { - await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(companyId, tx as unknown as Db), { + await createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({ companyId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { commentId: "fresh-input" }, agentId, source: "on_demand", triggerDetail: null, payload: null, requestedByActorType: "user", requestedByActorId: null, idempotencyKey: null, @@ -1565,8 +1555,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { paperclipOutputSecretRedactions: [{ fingerprintSha256: "b".repeat(64), material: { scheme: "synthetic" } }], }; const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { taskKey: "issue-1", ...history } }); - await db.transaction(async (tx) => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( - createAdmissionTransactionScope(companyId, tx as unknown as Db), { + await db.transaction(async (tx) => createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({ companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { taskKey: "issue-1", commentId: "fresh-input" }, source: "on_demand", triggerDetail: null, payload: null, @@ -1583,12 +1572,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { const otherCompanyId = await seedCompany(); const agentId = await seedAgent({ companyId }); const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { taskKey: "issue-1" } }); - const writer = createWakeAdmissionWriter(); await expect( db.transaction(async (tx) => { - const scope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db); - await writer.coalesceIntoActiveExecutionRun(scope, { + await createWakeAdmissionWriter(tx, otherCompanyId).coalesceIntoActiveExecutionRun({ companyId: otherCompanyId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", @@ -1610,15 +1597,11 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { expect(wakeRows).toHaveLength(0); }); - // Review test (d): a transaction-port call cannot use the root database - // executor. The module rejects a missing scope handle and a mismatched - // scope handle, and neither case writes a row. - it("rejects a missing transaction scope and a mismatched one, without writing through the root executor", async () => { + it("rejects input for another company before writing a deferred receipt", async () => { const companyId = await seedCompany(); const otherCompanyId = await seedCompany(); const agentId = await seedAgent({ companyId }); const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); - const writer = createWakeAdmissionWriter(); const newWakeInput = { companyId, agentId, @@ -1630,14 +1613,9 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { idempotencyKey: null, }; - await expect( - writer.insertNewDeferredWake(undefined as unknown as TransactionScope, newWakeInput), - ).rejects.toThrow(/transaction scope/); - await expect( db.transaction(async (tx) => { - const mismatchedScope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db); - await writer.insertNewDeferredWake(mismatchedScope, newWakeInput); + await createWakeAdmissionWriter(tx, otherCompanyId).insertNewDeferredWake(newWakeInput); }), ).rejects.toThrow(/different company/); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 7caa80b5f36..1a7463c04e3 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -42,7 +42,10 @@ import { legacyExecutionNeedsReconciliationWithEvidence } from "../../../service import { authorizeFailedChatRunRetryWake, FailedChatRunRetryAuthorizationError, + withDurableChatAdmission, + type DurableChatWakeupRequest, } from "../../../services/durable-chat-wakeup.js"; +import { isExternalChatWaitAuthorizationContention } from "../../../services/native-runtime/chat-attachment-reuse.js"; import { isRetiredExternalChatQuestionSource } from "../../../services/question-response-delivery.js"; import { HttpError } from "../../../errors.js"; import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokability.js"; @@ -73,7 +76,6 @@ import { parseObject, readNonEmptyString, } from "../domain/values.js"; -import { requireTransactionScopeTx, TransactionScope } from "../application/ports.js"; import type { DeferredWakeCandidate, InvokableAgentSnapshot, @@ -88,6 +90,7 @@ import type { WakeQueueTransaction, } from "../application/ports.js"; import type { FreshIssueRunStore } from "../application/fresh-issue-run.js"; +import type { IssueWakeAdmissionOutcome } from "../application/issue-wake-admission.js"; import type { RunSummary } from "../application/types.js"; const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; @@ -97,6 +100,12 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; type IssueRow = typeof issues.$inferSelect; +export type IssueAdmissionDbTransaction = Parameters[0]>[0]; +export type IssueAdmissionTarget = { + companyId: string; + issueId: string; + durableChatRequest?: DurableChatWakeupRequest; +}; function normalizeAgentNameKey(value: string | null | undefined): string | null { if (typeof value !== "string") return null; @@ -971,23 +980,17 @@ async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: ReleaseRun, iss return true; } -/** - * Builds the temporary transaction-scope handle the admission port needs. - * `heartbeat.ts` calls this through `createWakeQueue`'s own wrapper; it - * never builds a `TransactionScope` itself. - */ -export function createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope { - return TransactionScope.create(companyId, tx); +function assertAdmissionCompany(admittedCompanyId: string, companyId: string) { + if (companyId !== admittedCompanyId) { + throw new Error("wake-queue: admission is locked for a different company"); + } } -function requireAdmissionTx(scope: TransactionScope | null | undefined, companyId: string): Db { - return requireTransactionScopeTx(scope, companyId) as Db; -} - -export function createWakeAdmissionReader(): WakeAdmissionReader { +export function createWakeAdmissionReader(rawTx: IssueAdmissionDbTransaction, admittedCompanyId: string): WakeAdmissionReader { + const tx = rawTx as unknown as Db; return { - async matchesActiveWakeActor(scope, input) { - const tx = requireAdmissionTx(scope, input.companyId); + async matchesActiveWakeActor(input) { + assertAdmissionCompany(admittedCompanyId, input.companyId); if (!input.wakeupRequestId) return false; const actor = await tx .select({ @@ -1010,7 +1013,6 @@ export function createWakeAdmissionReader(): WakeAdmissionReader { ); }, async isSameExecutionAgent( - scope, { companyId, activeExecutionRunAgentId, @@ -1018,7 +1020,7 @@ export function createWakeAdmissionReader(): WakeAdmissionReader { agentNameKey, }, ) { - const tx = requireAdmissionTx(scope, companyId); + assertAdmissionCompany(admittedCompanyId, companyId); const executionAgent = await tx .select({ name: agents.name }) .from(agents) @@ -1038,10 +1040,9 @@ export function createWakeAdmissionReader(): WakeAdmissionReader { }, async findExistingDeferredWake( - scope, { companyId, agentId, issueId, chatConversationId, durableActor }, ) { - const tx = requireAdmissionTx(scope, companyId); + assertAdmissionCompany(admittedCompanyId, companyId); const row = await tx .select() .from(agentWakeupRequests) @@ -1098,10 +1099,11 @@ export function createWakeAdmissionReader(): WakeAdmissionReader { }; } -export function createWakeAdmissionWriter(): WakeAdmissionWriter { +export function createWakeAdmissionWriter(rawTx: IssueAdmissionDbTransaction, admittedCompanyId: string): WakeAdmissionWriter { + const tx = rawTx as unknown as Db; return { - async coalesceIntoActiveExecutionRun(scope, input) { - const tx = requireAdmissionTx(scope, input.companyId); + async coalesceIntoActiveExecutionRun(input) { + assertAdmissionCompany(admittedCompanyId, input.companyId); const now = new Date(); const [candidate] = await tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, input.activeExecutionRunId), eq(heartbeatRuns.companyId, input.companyId), @@ -1173,8 +1175,8 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { return mergedRun as unknown as Record; }, - async mergeIntoExistingDeferredWake(scope, input) { - const tx = requireAdmissionTx(scope, input.companyId); + async mergeIntoExistingDeferredWake(input) { + assertAdmissionCompany(admittedCompanyId, input.companyId); if (input.manualUserWakeActorId) await notifyChatInboundWakeNotices(tx, { companyId: input.companyId }); const [prior] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, input.existingDeferredWakeId), @@ -1256,8 +1258,8 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { } }, - async insertNewDeferredWake(scope, input) { - const tx = requireAdmissionTx(scope, input.companyId); + async insertNewDeferredWake(input) { + assertAdmissionCompany(admittedCompanyId, input.companyId); await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.insert(agentWakeupRequests).values({ ...input.durableReceipt, @@ -1276,6 +1278,32 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { }; } +export function createPostgresIssueWakeAdmission(db: Db) { + return async function withIssueAdmission( + { companyId, issueId, durableChatRequest }: IssueAdmissionTarget, + evaluate: (tx: IssueAdmissionDbTransaction, stores: { + reader: WakeAdmissionReader; + writer: WakeAdmissionWriter; + fresh: FreshIssueRunStore; + }) => Promise>, + ): Promise | null> { + try { + return await db.transaction(withDurableChatAdmission(durableChatRequest, async (tx) => { + // Mentions may target another agent, but every admission must share the issue lock. + await tx.execute(sql`select id from issues where id = ${issueId} and company_id = ${companyId} for update`); + return evaluate(tx, { + reader: createWakeAdmissionReader(tx, companyId), + writer: createWakeAdmissionWriter(tx, companyId), + fresh: createFreshIssueRunStore(tx, companyId), + }); + })); + } catch (error) { + if (isExternalChatWaitAuthorizationContention(error)) return null; + throw error; + } + }; +} + export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter { return { async withIssueExecutionLock(input, fn): Promise { @@ -1539,10 +1567,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd } export function createFreshIssueRunStore( - scope: TransactionScope, + rawTx: IssueAdmissionDbTransaction, companyId: string, ): FreshIssueRunStore { - const tx = requireAdmissionTx(scope, companyId); + const tx = rawTx as unknown as Db; return { async saveReceipt(input) { if (input.kind === "insert") { diff --git a/server/src/modules/wake-queue/application/fresh-issue-run.ts b/server/src/modules/wake-queue/application/fresh-issue-run.ts index a5c7da4a4f3..1ef05f937ba 100644 --- a/server/src/modules/wake-queue/application/fresh-issue-run.ts +++ b/server/src/modules/wake-queue/application/fresh-issue-run.ts @@ -103,7 +103,7 @@ function isOrdinaryBoardComment(wake: DeferredCommentCandidate): boolean { queuedCommentIdsFromWakePayload(payload).length > 0; } -/** Admit one new run and its saved inputs using the caller's existing issue lock. */ +/** Admit one new run and its saved inputs under the issue admission lock. */ export function createFreshIssueRunAdmission(store: FreshIssueRunStore) { return async function admitFreshIssueRun(input: FreshIssueRunInput): Promise { const { receipt, durableReceipt, durableChatReceiptId, executionWaitRequestId, issue, contextSnapshot } = input; diff --git a/server/src/modules/wake-queue/application/issue-wake-admission.test.ts b/server/src/modules/wake-queue/application/issue-wake-admission.test.ts new file mode 100644 index 00000000000..4ae47b68427 --- /dev/null +++ b/server/src/modules/wake-queue/application/issue-wake-admission.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { admitIssueWake, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./issue-wake-admission.js"; + +const queuedRun = { id: "run", companyId: "company", agentId: "agent", status: "queued", + invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "wake" }; +type Run = typeof queuedRun; +const runningRun = { ...queuedRun, status: "running" }; + +function recordEffects(trace: string[]): IssueWakeAdmissionEffects { + return { + saved: () => { trace.push("saved"); }, + publishIssueActivity: () => { trace.push("issue activity"); }, + executeIssueActions: async () => { trace.push("issue actions"); }, + cancelledRuns: [{ ...queuedRun, status: "cancelled" }], + emitCancelledRun: (run) => { + trace.push(run.status); + return new Promise(() => {}); + }, + readRun: async (runId) => { trace.push(`read ${runId}`); return runningRun; }, + publishQueuedRun: () => { trace.push("queued event"); }, + attachSetupRequest: async () => { trace.push("attach setup"); }, + dispatch: async () => { trace.push("dispatch"); }, + }; +} + +describe("issue wake admission handoff", () => { + it("waits for commit before publishing or dispatching and leaves telemetry in the background", async () => { + const trace: string[] = []; + let commit!: (outcome: IssueWakeAdmissionOutcome) => void; + const transaction = new Promise>((resolve) => { commit = resolve; }); + const result = admitIssueWake(() => transaction, recordEffects(trace)); + expect(trace).toEqual([]); + + commit({ kind: "queued", run: queuedRun }); + await expect(result).resolves.toBe(queuedRun); + expect(trace).toEqual(["saved", "issue activity", "issue actions", "cancelled", "queued event", "attach setup", "dispatch"]); + }); + + it.each<{ outcome: IssueWakeAdmissionOutcome; returned: Run | null; handoff: string[] }>([ + { outcome: { kind: "coalesced", run: runningRun }, returned: runningRun, handoff: ["attach setup", "dispatch"] }, + { outcome: { kind: "replayed", run: queuedRun }, returned: queuedRun, handoff: ["attach setup", "dispatch"] }, + { outcome: { kind: "replayed", run: runningRun }, returned: runningRun, handoff: [] }, + { outcome: { kind: "durable", receipt: { runId: "run" } }, returned: runningRun, handoff: ["read run"] }, + { outcome: { kind: "durable", receipt: { runId: null } }, returned: null, handoff: [] }, + { outcome: { kind: "deferred", admitted: true }, returned: null, handoff: [] }, + { outcome: { kind: "skipped" }, returned: null, handoff: [] }, + ])("hands off $outcome.kind with the persisted run or receipt", async ({ outcome, returned, handoff }) => { + const trace: string[] = []; + await expect(admitIssueWake(async () => outcome, recordEffects(trace))).resolves.toBe(returned); + const saved = outcome.kind === "deferred" || outcome.kind === "skipped" ? [] : ["saved"]; + expect(trace).toEqual([...saved, "issue activity", "issue actions", "cancelled", ...handoff]); + }); + + it("discards all buffered work when commit rejects", async () => { + const trace: string[] = []; + const error = new Error("commit failed"); + await expect(admitIssueWake(() => Promise.reject(error), recordEffects(trace))).rejects.toBe(error); + expect(trace).toEqual([]); + }); + + it("surfaces post-commit failures even when they look like authorization contention", async () => { + const trace: string[] = []; + const effects = recordEffects(trace); + const error = Object.assign(new Error("post-commit failure"), { code: "55P03" }); + effects.executeIssueActions = async () => { throw error; }; + await expect(admitIssueWake(async () => ({ kind: "queued", run: queuedRun }), effects)).rejects.toBe(error); + expect(trace).toEqual(["saved", "issue activity"]); + }); +}); diff --git a/server/src/modules/wake-queue/application/issue-wake-admission.ts b/server/src/modules/wake-queue/application/issue-wake-admission.ts new file mode 100644 index 00000000000..fac3a4e424f --- /dev/null +++ b/server/src/modules/wake-queue/application/issue-wake-admission.ts @@ -0,0 +1,47 @@ +import type { RunSummary } from "./types.js"; + +export type IssueWakeAdmissionOutcome = + | { kind: "queued" | "coalesced" | "replayed"; run: Run } + | { kind: "durable"; receipt: { runId: string | null } } + | { kind: "deferred"; admitted?: boolean } + | { kind: "skipped" }; + +export type IssueWakeAdmissionEffects = { + saved?(): void; + publishIssueActivity(): void; + executeIssueActions(): Promise; + cancelledRuns: readonly Run[]; + emitCancelledRun(run: Run): Promise; + readRun(runId: string): Promise; + publishQueuedRun(run: RunSummary): void; + attachSetupRequest(): Promise; + dispatch(): Promise; +}; + +/** Hand off only a committed admission. A contended authorization rolls back. */ +export async function admitIssueWake( + commit: () => Promise | null>, + effects: IssueWakeAdmissionEffects, +): Promise { + const outcome = await commit(); + if (!outcome) return null; + + if (outcome.kind !== "deferred" && outcome.kind !== "skipped") effects.saved?.(); + effects.publishIssueActivity(); + await effects.executeIssueActions(); + + // Cancellation telemetry does not gate dispatch or the admission result. + for (const run of effects.cancelledRuns) void effects.emitCancelledRun(run); + + if (outcome.kind === "durable") { + return outcome.receipt.runId ? effects.readRun(outcome.receipt.runId) : null; + } + if (outcome.kind === "deferred" || outcome.kind === "skipped") return null; + + if (outcome.kind === "queued") effects.publishQueuedRun(outcome.run); + if (outcome.kind !== "replayed" || outcome.run.status === "queued") { + await effects.attachSetupRequest(); + await effects.dispatch(); + } + return outcome.run; +} diff --git a/server/src/modules/wake-queue/application/ports.ts b/server/src/modules/wake-queue/application/ports.ts index 2ee3256260f..f6061ea37a0 100644 --- a/server/src/modules/wake-queue/application/ports.ts +++ b/server/src/modules/wake-queue/application/ports.ts @@ -305,49 +305,6 @@ export interface RecoveryEscalationPort { export type { PostCommitEffect, ReleaseOutcome }; -/** - * Temporary port: `heartbeat.ts` still opens and owns the transaction that - * admits a wake behind an active issue execution; the module does not own - * that transaction yet. A later change will decompose `enqueueWakeup` so - * the module owns the transaction itself. That change removes this handle - * and replaces it with a transaction the module opens on its own. - * - * Only this module builds a scope. `createWakeQueue` exposes a method that - * builds one for a caller outside the module. That caller receives an - * opaque handle back and never touches this class directly. - */ -export class TransactionScope { - private constructor( - private readonly companyId: string, - private readonly rawTx: unknown, - ) {} - - static create(companyId: string, rawTx: unknown): TransactionScope { - return new TransactionScope(companyId, rawTx); - } - - /** Returns the bound transaction only when `companyId` matches the scope's own company. */ - requireTx(companyId: string): unknown { - if (companyId !== this.companyId) { - throw new Error( - "wake-queue: the transaction scope belongs to a different company than the requested write", - ); - } - return this.rawTx; - } -} - -/** Reads a scope's bound transaction. Rejects a missing scope with the same clear error as a mismatched one; never falls back to any other executor. */ -export function requireTransactionScopeTx( - scope: TransactionScope | null | undefined, - companyId: string, -): unknown { - if (!scope) { - throw new Error("wake-queue: this call carries no transaction scope"); - } - return scope.requireTx(companyId); -} - /** The active execution run a new wake arrives behind. */ export type WakeAdmissionActiveExecutionRun = { id: string; @@ -422,11 +379,10 @@ export type AdmitWakeBehindIssueExecutionResult = | { kind: "coalesced"; run: Record } | { kind: "deferred" }; -/** Read-only lookups the admission use case needs, each scoped to a company. */ +/** Read-only admission lookups bound to the issue admission transaction. */ export interface WakeAdmissionReader { /** A durable incoming request may share an active run only with the exact actor on its persisted wake receipt. */ matchesActiveWakeActor( - scope: TransactionScope, input: { companyId: string; wakeupRequestId: string | null; @@ -436,7 +392,6 @@ export interface WakeAdmissionReader { ): Promise; /** True when the active execution run's agent and this wake's own agent share an execution-agent-name key. */ isSameExecutionAgent( - scope: TransactionScope, input: { companyId: string; activeExecutionRunAgentId: string; @@ -445,7 +400,6 @@ export interface WakeAdmissionReader { }, ): Promise; findExistingDeferredWake( - scope: TransactionScope, input: { companyId: string; agentId: string; @@ -461,7 +415,6 @@ export interface WakeAdmissionReader { export interface WakeAdmissionWriter { /** Merges the wake's context into the active execution run and records the wake as coalesced. Returns the updated run row. */ coalesceIntoActiveExecutionRun( - scope: TransactionScope, input: { companyId: string; activeExecutionRunId: string; @@ -479,7 +432,6 @@ export interface WakeAdmissionWriter { ): Promise | null>; /** Merges the wake's context into an already-queued deferred wake, guarded by its current status. */ mergeIntoExistingDeferredWake( - scope: TransactionScope, input: { companyId: string; existingDeferredWakeId: string; @@ -493,7 +445,6 @@ export interface WakeAdmissionWriter { ): Promise; /** Queues a new deferred wake behind the active execution run. */ insertNewDeferredWake( - scope: TransactionScope, input: { companyId: string; agentId: string; diff --git a/server/src/modules/wake-queue/application/use-cases.test.ts b/server/src/modules/wake-queue/application/use-cases.test.ts index 8cf2e851f00..a8b20be0c1b 100644 --- a/server/src/modules/wake-queue/application/use-cases.test.ts +++ b/server/src/modules/wake-queue/application/use-cases.test.ts @@ -11,7 +11,6 @@ import type { RecoveryEscalationPort, RunSnapshot, RunSummary, - TransactionScope, WakeAdmissionActiveExecutionRun, WakeAdmissionHeartbeatHelpers, WakeAdmissionReader, @@ -914,9 +913,6 @@ const ACTIVE_EXECUTION_RUN: WakeAdmissionActiveExecutionRun = { contextSnapshot: { taskKey: "issue-1" }, }; -// The scope is opaque to the use case; the fakes below never inspect it. -const SCOPE = {} as TransactionScope; - function admissionInput( overrides: Partial = {}, ): AdmitWakeBehindIssueExecutionInput { @@ -1011,7 +1007,6 @@ describe("admitWakeBehindIssueExecution", () => { }; const result = await admit( - SCOPE, admissionInput({ activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status }, allowRunCoalescing: false, @@ -1025,7 +1020,6 @@ describe("admitWakeBehindIssueExecution", () => { expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); expect(writer.mergeIntoExistingDeferredWake).not.toHaveBeenCalled(); expect(writer.insertNewDeferredWake).toHaveBeenCalledExactlyOnceWith( - SCOPE, expect.objectContaining({ durableReceipt, payload: { @@ -1047,7 +1041,7 @@ describe("admitWakeBehindIssueExecution", () => { } : null), }); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() }); - await admit(SCOPE, admissionInput({ + await admit(admissionInput({ contextSnapshot: location === "incoming" ? { interactionId: "approval" } : {}, activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status: "queued", contextSnapshot: location === "queued" ? { interactionId: "approval" } : {} }, @@ -1066,7 +1060,7 @@ describe("admitWakeBehindIssueExecution", () => { const writer = createFakeAdmissionWriter(); const reader = createFakeAdmissionReader(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() }); - await admit(SCOPE, admissionInput({ + await admit(admissionInput({ contextSnapshot: incoming ? { chatConversationId: incoming } : {}, activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status: "queued", contextSnapshot: active ? { chatConversationId: active } : {} }, @@ -1074,7 +1068,7 @@ describe("admitWakeBehindIssueExecution", () => { expect(writer.coalesceIntoActiveExecutionRun).toHaveBeenCalledTimes(coalesces ? 1 : 0); if (!coalesces) { // The separated reply still merges with its own conversation's deferred wake. - expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith(SCOPE, + expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ chatConversationId: incoming })); expect(writer.insertNewDeferredWake).toHaveBeenCalledOnce(); } @@ -1097,7 +1091,6 @@ describe("admitWakeBehindIssueExecution", () => { }); const result = await admit( - SCOPE, admissionInput({ activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, @@ -1111,7 +1104,6 @@ describe("admitWakeBehindIssueExecution", () => { expect(result).toEqual({ kind: "deferred" }); expect(reader.matchesActiveWakeActor).toHaveBeenCalledExactlyOnceWith( - SCOPE, { companyId: "company-1", wakeupRequestId: "active-wake", @@ -1121,7 +1113,6 @@ describe("admitWakeBehindIssueExecution", () => { ); expect(writer.coalesceIntoActiveExecutionRun).not.toHaveBeenCalled(); expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith( - SCOPE, { companyId: "company-1", agentId: "wake-agent", @@ -1131,7 +1122,6 @@ describe("admitWakeBehindIssueExecution", () => { }, ); expect(writer.insertNewDeferredWake).toHaveBeenCalledExactlyOnceWith( - SCOPE, expect.objectContaining({ durableReceipt }), ); }); @@ -1151,7 +1141,6 @@ describe("admitWakeBehindIssueExecution", () => { }); const result = await admit( - SCOPE, admissionInput({ activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, @@ -1163,7 +1152,6 @@ describe("admitWakeBehindIssueExecution", () => { expect(result).toEqual({ kind: "coalesced", run: { id: "merged-run-1" } }); expect(reader.matchesActiveWakeActor).toHaveBeenCalledExactlyOnceWith( - SCOPE, { companyId: "company-1", wakeupRequestId: "active-wake", @@ -1174,7 +1162,6 @@ describe("admitWakeBehindIssueExecution", () => { expect( writer.coalesceIntoActiveExecutionRun, ).toHaveBeenCalledExactlyOnceWith( - SCOPE, expect.objectContaining({ durableReceipt }), ); expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); @@ -1204,7 +1191,6 @@ describe("admitWakeBehindIssueExecution", () => { }); const result = await admit( - SCOPE, admissionInput({ durableReceipt, reason: "question_response", @@ -1216,7 +1202,6 @@ describe("admitWakeBehindIssueExecution", () => { expect( writer.mergeIntoExistingDeferredWake, ).toHaveBeenCalledExactlyOnceWith( - SCOPE, expect.objectContaining({ existingDeferredWakeId: "existing-deferred-wake", nextCoalescedCount: 3, @@ -1256,10 +1241,10 @@ describe("admitWakeBehindIssueExecution", () => { })) }); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) }); - await expect(admit(SCOPE, admissionInput({ wakeCommentId: "second-comment", requestedByActorId: "second-user", + await expect(admit(admissionInput({ wakeCommentId: "second-comment", requestedByActorId: "second-user", payload: { commentId: "second-comment", coalescedIntoWakeupRequestId: "forged-target" }, }))).resolves.toEqual({ kind: "deferred" }); - const input = vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1]; + const input = vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0]; expect(input.coalescedReceipt).toEqual({ agentId: "wake-agent", source: "on_demand", triggerDetail: null, reason: null, payload: { issueId: "issue-1", commentId: "second-comment", coalescedIntoWakeupRequestId: "existing-deferred-wake" }, requestedByActorType: "user", requestedByActorId: "second-user", idempotencyKey: null, runId: null }); @@ -1275,8 +1260,8 @@ describe("admitWakeBehindIssueExecution", () => { reader: createFakeAdmissionReader({ findExistingDeferredWake: vi.fn(async () => ({ id: "saved-wake", runId: null, payload: { issueId: "issue-1" }, deferredContext: {}, coalescedCount: 0, })) }), helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) }); - await admit(SCOPE, admissionInput({ durableReceipt, payload: { text: "saved input" } })); - expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1].coalescedReceipt).toEqual(expect.objectContaining({ + await admit(admissionInput({ durableReceipt, payload: { text: "saved input" } })); + expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0].coalescedReceipt).toEqual(expect.objectContaining({ ...durableReceipt, payload: { text: "saved input", issueId: "issue-1", coalescedIntoWakeupRequestId: "saved-wake" }, })); }); @@ -1287,8 +1272,8 @@ describe("admitWakeBehindIssueExecution", () => { reader: createFakeAdmissionReader({ findExistingDeferredWake: vi.fn(async () => ({ id: "existing-deferred-wake", runId: null, payload: { issueId: "issue-1" }, deferredContext: {}, coalescedCount: 1, })) }), helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) }); - await admit(SCOPE, admissionInput({ wakeCommentId: "second-comment", requestedByActorType: actor })); - expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1]).not.toHaveProperty("coalescedReceipt"); + await admit(admissionInput({ wakeCommentId: "second-comment", requestedByActorType: actor })); + expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0]).not.toHaveProperty("coalescedReceipt"); }); it("gives manual input its own run boundary even when the active receipt has the same requester", async () => { @@ -1296,7 +1281,7 @@ describe("admitWakeBehindIssueExecution", () => { const admit = createAdmitWakeBehindIssueExecution({ reader: createFakeAdmissionReader(), writer, helpers: createFakeAdmissionHelpers(), }); - expect(await admit(SCOPE, admissionInput({ payload: { issueId: "issue-1", manualUserWake: true } }))) + expect(await admit(admissionInput({ payload: { issueId: "issue-1", manualUserWake: true } }))) .toEqual({ kind: "deferred" }); expect(writer.coalesceIntoActiveExecutionRun).not.toHaveBeenCalled(); expect(writer.insertNewDeferredWake).toHaveBeenCalledTimes(1); @@ -1314,7 +1299,7 @@ describe("admitWakeBehindIssueExecution", () => { helpers, }); - expect(await admit(SCOPE, admissionInput())).toEqual({ + expect(await admit(admissionInput())).toEqual({ kind: "coalesced", run: { id: "merged-run-1" }, }); @@ -1328,7 +1313,7 @@ describe("admitWakeBehindIssueExecution", () => { const helpers = createFakeAdmissionHelpers(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers }); - const result = await admit(SCOPE, admissionInput()); + const result = await admit(admissionInput()); expect(result).toEqual({ kind: "coalesced", run: { id: "merged-run-1" } }); expect(writer.coalesceIntoActiveExecutionRun).toHaveBeenCalledTimes(1); @@ -1342,7 +1327,7 @@ describe("admitWakeBehindIssueExecution", () => { const helpers = createFakeAdmissionHelpers(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers }); - const result = await admit(SCOPE, admissionInput()); + const result = await admit(admissionInput()); expect(result.kind).toBe("coalesced"); expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); @@ -1353,10 +1338,10 @@ describe("admitWakeBehindIssueExecution", () => { const reader = createFakeAdmissionReader(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() }); const durableReceipt = { id: "incoming-source", requestedAt: new Date("2026-11-04T00:00:00Z") }; - expect(await admit(SCOPE, admissionInput({ durableReceipt, wakeCommentId: "new-comment", + expect(await admit(admissionInput({ durableReceipt, wakeCommentId: "new-comment", contextSnapshot: { issueId: "issue-1", wakeCommentId: "new-comment" }, payload: { issueId: "issue-1", commentId: "new-comment" }, }))).toEqual({ kind: "deferred" }); - expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(SCOPE, expect.objectContaining({ + expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(expect.objectContaining({ companyId: "company-1", agentId: "wake-agent", durableReceipt, requestedByActorType: "user", requestedByActorId: "user-1", payload: expect.objectContaining({ issueId: "issue-1", commentId: "new-comment" }), @@ -1366,7 +1351,7 @@ describe("admitWakeBehindIssueExecution", () => { it("merges into the existing deferred wake when the policy returns a merge target", async () => { const mergeIntoExistingDeferredWake = vi.fn( - async (_scope: TransactionScope, _input: Parameters[1]) => {}, + async (_input: Parameters[0]) => {}, ); const writer = createFakeAdmissionWriter({ mergeIntoExistingDeferredWake }); const reader = createFakeAdmissionReader({ @@ -1381,11 +1366,11 @@ describe("admitWakeBehindIssueExecution", () => { const helpers = createFakeAdmissionHelpers(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers }); - const result = await admit(SCOPE, admissionInput()); + const result = await admit(admissionInput()); expect(result).toEqual({ kind: "deferred" }); expect(mergeIntoExistingDeferredWake).toHaveBeenCalledTimes(1); - const call = mergeIntoExistingDeferredWake.mock.calls[0]![1]; + const call = mergeIntoExistingDeferredWake.mock.calls[0]![0]; expect(call.existingDeferredWakeId).toBe("deferred-1"); expect(call.nextCoalescedCount).toBe(3); expect(call.mergedPayload.foo).toBe("bar"); @@ -1399,7 +1384,7 @@ describe("admitWakeBehindIssueExecution", () => { const helpers = createFakeAdmissionHelpers(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers }); - const result = await admit(SCOPE, admissionInput()); + const result = await admit(admissionInput()); expect(result).toEqual({ kind: "deferred" }); expect(writer.insertNewDeferredWake).toHaveBeenCalledTimes(1); @@ -1413,7 +1398,7 @@ describe("admitWakeBehindIssueExecution", () => { const helpers = createFakeAdmissionHelpers(); const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers }); - const result = await admit(SCOPE, admissionInput({ liveRunExecutions: { has: () => false } })); + const result = await admit(admissionInput({ liveRunExecutions: { has: () => false } })); expect(result).toEqual({ kind: "proceed" }); expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts index ccd0817f78b..7cf7c23ecf5 100644 --- a/server/src/modules/wake-queue/application/use-cases.ts +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -26,7 +26,6 @@ import type { RecoveryEscalationPort, ReleaseTransactionResult, RunSnapshot, - TransactionScope, WakeAdmissionActiveExecutionRun, WakeAdmissionHeartbeatHelpers, WakeAdmissionReader, @@ -868,7 +867,6 @@ export function createAdmitWakeBehindIssueExecution(deps: { helpers: WakeAdmissionHeartbeatHelpers; }) { return async function admitWakeBehindIssueExecution( - scope: TransactionScope, input: AdmitWakeBehindIssueExecutionInput, ): Promise { const manualUserWakeActorId = input.payload?.manualUserWake === true @@ -877,7 +875,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { (input.requestedByActorType !== "user" || !manualUserWakeActorId)) { throw new Error("wake-queue: manual wake requires an authenticated user"); } - const isSameExecutionAgent = await deps.reader.isSameExecutionAgent(scope, { + const isSameExecutionAgent = await deps.reader.isSameExecutionAgent({ companyId: input.companyId, activeExecutionRunAgentId: input.activeExecutionRun.agentId, issueExecutionAgentNameKey: input.issueExecutionAgentNameKey, @@ -909,7 +907,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { const sameDurableActor = !input.durableReceipt || - (await deps.reader.matchesActiveWakeActor(scope, { + (await deps.reader.matchesActiveWakeActor({ companyId: input.companyId, wakeupRequestId: availableActiveExecutionRun?.wakeupRequestId ?? null, requestedByActorType: input.requestedByActorType, @@ -940,7 +938,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { target.status === "queued" || target.status === "scheduled_retry", }, ); - const run = await deps.writer.coalesceIntoActiveExecutionRun(scope, { + const run = await deps.writer.coalesceIntoActiveExecutionRun({ companyId: input.companyId, activeExecutionRunId: target.id, activeExecutionRunStatus: target.status, @@ -962,7 +960,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { const existingDeferred = (interactionReceipt || input.allowRunCoalescing === false) ? null - : await deps.reader.findExistingDeferredWake(scope, { + : await deps.reader.findExistingDeferredWake({ companyId: input.companyId, agentId: input.agentId, issueId: input.issueId, @@ -995,7 +993,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { issueId: input.issueId, [DEFERRED_WAKE_CONTEXT_KEY]: mergedDeferredContext, }; - await deps.writer.mergeIntoExistingDeferredWake(scope, { + await deps.writer.mergeIntoExistingDeferredWake({ companyId: input.companyId, existingDeferredWakeId: existingDeferred.id, mergedPayload, @@ -1031,7 +1029,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { issueId: input.issueId, [DEFERRED_WAKE_CONTEXT_KEY]: input.contextSnapshot, }; - await deps.writer.insertNewDeferredWake(scope, { + await deps.writer.insertNewDeferredWake({ companyId: input.companyId, agentId: input.agentId, source: input.source, diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 2ad2b1247ab..16e1ebaa0b1 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -1,14 +1,14 @@ -import type { Db } from "@paperclipai/db"; +import type { Db, heartbeatRuns } from "@paperclipai/db"; import { - createAdmissionTransactionScope as buildAdmissionTransactionScope, - createFreshIssueRunStore, + createPostgresIssueWakeAdmission, createPostgresWakeQueueAdapter, - createWakeAdmissionReader, - createWakeAdmissionWriter, + type IssueAdmissionDbTransaction, + type IssueAdmissionTarget, } from "./adapters/postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js"; import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js"; +import { admitIssueWake, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./application/issue-wake-admission.js"; import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js"; import { createDiscardQueuedComment, @@ -19,7 +19,6 @@ import type { IssueSnapshot, RecoveryEscalationPort, RunSnapshot, - TransactionScope, WakeAdmissionHeartbeatHelpers, WakeQueueHost, } from "./application/ports.js"; @@ -37,9 +36,8 @@ export type { RunSnapshot, RecoveryEscalationPort, ReleaseRecoveryBlockedNoticeKind, - TransactionScope, } from "./application/ports.js"; -export type { AdmitWakeBehindIssueExecutionInput, AdmitWakeBehindIssueExecutionResult, ReleaseIssueExecutionInput } from "./application/use-cases.js"; +export type { ReleaseIssueExecutionInput } from "./application/use-cases.js"; export { QueuedCommentMutationError, QueuedCommentMutationForbiddenError, @@ -78,45 +76,53 @@ export type WakeQueueDeps = { recovery: RecoveryEscalationPort; }; -/** - * Composes the wake-queue module: the Postgres adapter (which owns the - * release transaction) and the release use case. `heartbeat.ts` holds the - * only caller: it builds one instance per process next to - * `createRunDispatch(db)` and delegates `releaseIssueExecutionAndPromote`'s - * body to `releaseIssueExecution`. - * - * The admission half is temporary: `heartbeat.ts` still opens and owns the - * transaction that admits a wake behind an active issue execution, so it - * builds a `TransactionScope` through `createAdmissionTransactionScope` - * before it calls `admitWakeBehindIssueExecution`. - */ +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type LockedIssueAdmission = { + admitWakeBehindIssueExecution: ReturnType; + admitFreshIssueRun: ReturnType>; +}; + +/** Composes the issue admission and release workflows with their Postgres locks. */ export function createWakeQueue(db: Db, deps: WakeQueueDeps) { const issueLock = createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: deps.resolveResponsibleUserId, getRoutineEnv: deps.getRoutineEnv, resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup, }); + const withIssueAdmission = createPostgresIssueWakeAdmission(db); return { releaseIssueExecution: createReleaseIssueExecution({ issueLock, recovery: deps.recovery }), - admitWakeBehindIssueExecution: createAdmitWakeBehindIssueExecution({ - reader: createWakeAdmissionReader(), - writer: createWakeAdmissionWriter(), - helpers: deps.wakeAdmissionHelpers, - }), - createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope { - return buildAdmissionTransactionScope(companyId, tx); + admitIssueWake( + { companyId, issueId, durableChatRequest }: IssueAdmissionTarget, + // Specialized heartbeat gates and continuation writes still use this locked transaction. + evaluateHeartbeatPolicy: (tx: IssueAdmissionDbTransaction, admission: LockedIssueAdmission) => Promise>, + effects: IssueWakeAdmissionEffects, + ) { + return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { reader, writer, fresh }) => { + const admitBusy = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: deps.wakeAdmissionHelpers }); + const admitFresh = createFreshIssueRunAdmission(fresh); + return evaluateHeartbeatPolicy(tx, { + async admitWakeBehindIssueExecution(input) { + if (input.companyId !== companyId || input.issueId !== issueId || input.contextSnapshot.issueId !== issueId) { + throw new Error("wake-queue: admission does not match the locked company and issue"); + } + return admitBusy(input); + }, + async admitFreshIssueRun(input) { + if (input.issue.id !== issueId || input.contextSnapshot.issueId !== issueId) { + throw new Error("wake-queue: admission does not match the locked issue"); + } + return admitFresh(input); + }, + }); + }), effects); }, }; } export type WakeQueue = ReturnType; -/** Uses the transaction that already owns the issue's admission lock. */ -export function bindFreshIssueRunAdmission(companyId: string, tx: Db) { - return createFreshIssueRunAdmission(createFreshIssueRunStore(buildAdmissionTransactionScope(companyId, tx), companyId)); -} - /** * Composes the three queued-comment queue mutations (edit, reorder, * discard): the Postgres adapter, which owns the one transaction each diff --git a/server/src/services/agent-interaction-delivery.test.ts b/server/src/services/agent-interaction-delivery.test.ts index 1514f9d7362..c3ae02d30e0 100644 --- a/server/src/services/agent-interaction-delivery.test.ts +++ b/server/src/services/agent-interaction-delivery.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueThreadInteractions, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, type Db } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueThreadInteractions, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { filterRunMessageInteractionDelivery, hasAutomaticInteractionDeliveryHold, preparedRunInteractionDeliveryDenied, recheckQueuedCommentInteractionDelivery, recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "./agent-interaction-delivery.js"; @@ -9,7 +9,7 @@ import { createHeartbeatQueue } from "./heartbeat/queue.js"; import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js"; import { withPluginInteractionOrigin } from "./plugin-interaction-context.js"; import { recoveryService } from "./recovery/service.js"; -import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; const support = await getEmbeddedPostgresTestSupport(); describe.skipIf(!support.supported)("pending agent interaction delivery", () => { @@ -113,8 +113,7 @@ describe.skipIf(!support.supported)("pending agent interaction delivery", () => const context = { issueId: f.issueId, wakeReason: reason, ...(assignmentFirst ? { wakeCommentId: messageId, wakeCommentIds: [messageId] } : {}) }; const actorType = assignmentFirst ? "user" : "agent", actorId = assignmentFirst ? "human" : f.sender; const payload = withInteractionDeliveryOrigin({ issueId: f.issueId, _paperclipWakeContext: context }, actorType, actorId); - await db.transaction(tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( - createAdmissionTransactionScope(f.companyId, tx as unknown as Db), { + await db.transaction(tx => createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({ companyId: f.companyId, agentId: f.recipient, activeExecutionRunId: f.run.id, activeExecutionRunStatus: "queued", mergedContextSnapshot: { ...context, wakeCommentId: messageId, wakeCommentIds: [messageId] }, source: "assignment", triggerDetail: "system", payload, requestedByActorType: actorType, diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 8808e4871e9..9a7e8717144 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -12,11 +12,11 @@ import { createReleaseIssueExecution } from "../../modules/wake-queue/applicatio import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; import { createWakeQueue } from "../../modules/wake-queue/index.js"; -import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; import { settleConversationTurn } from "../agent-conversations.js"; import { chatCompletionDeliveryService, recordChatCompletion, recordChatHandoff } from "../chat-completion-delivery.js"; +import { createDurableChatWakeupRequest } from "../durable-chat-wakeup.js"; import { instanceSettingsService } from "../instance-settings.js"; import { createHeartbeatRunControl } from "./run-control.js"; import { callbacks as runControlCallbacks } from "./__tests__/run-control-fixtures.js"; @@ -255,6 +255,60 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { ...f, agent, service }; } + it.each([ + ["fresh", "authorization contention"], ["busy", "authorization contention"], + ["fresh", "wrong locked issue"], ["busy", "wrong locked issue"], + ] as const)("rolls back a %s chat admission on %s without publishing", async (kind, failure) => { + const { user, company, agent, issue, deps, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" }) + .where(eq(issues.id, issue.id)); + if (kind === "busy") { + const [active] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + status: "running", invocationSource: "assignment", contextSnapshot: { issueId: issue.id } }).returning(); + await db.update(issues).set({ executionRunId: active.id, executionAgentNameKey: agent.name, executionLockedAt: new Date() }) + .where(eq(issues.id, issue.id)); + deps.liveRunExecutions.has.mockImplementation((id) => id === active.id); + } + const [comment] = await db.insert(issueComments).values({ companyId: company.id, issueId: issue.id, + authorType: "user", authorUserId: user.id, body: "Please continue" }).returning(); + const id = randomUUID(); + const authorize = vi.fn(async () => {}); + const onAdmitted = vi.fn(async (tx: Db, { runId }: { runId: string | null }) => { + const [receipt] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)); + expect(receipt).toMatchObject({ status: kind === "fresh" ? "queued" : "deferred_issue_execution", runId }); + expect(Boolean(runId)).toBe(kind === "fresh"); + throw Object.assign(new Error("authorization lock was held"), { code: "55P03" }); + }); + const request = createDurableChatWakeupRequest({ id, companyId: company.id, issueId: issue.id, agentId: agent.id, + commentId: comment.id, requestedByActorType: "user", requestedByActorId: user.id, requestedAt: new Date(), authorize, onAdmitted }); + const [otherIssue] = failure === "wrong locked issue" ? await db.insert(issues).values({ + companyId: company.id, assigneeAgentId: agent.id, title: "Another task", + }).returning() : []; + const publishIssueActivity = vi.fn(); + const admit = deps.wakeQueue.admitIssueWake.getMockImplementation()!; + deps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admit(target, + (tx, admission) => evaluate(tx, otherIssue ? { + admitFreshIssueRun: (input) => admission.admitFreshIssueRun({ ...input, + issue: { ...input.issue, id: otherIssue.id }, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), + admitWakeBehindIssueExecution: (input) => admission.admitWakeBehindIssueExecution({ ...input, + issueId: otherIssue.id, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), + } : admission), { ...effects, publishIssueActivity })); + + const wake = queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "External chat message received", + requestedByActorType: "user", requestedByActorId: user.id, allowRunCoalescing: false, durableChatRequest: request, + payload: { issueId: issue.id, commentId: comment.id, mutation: "chat_message_received" }, + }); + if (otherIssue) await expect(wake).rejects.toThrow(/does not match the locked/); + else await expect(wake).resolves.toBeNull(); + expect(authorize).toHaveBeenCalledTimes(1); + expect(onAdmitted).toHaveBeenCalledTimes(otherIssue ? 0 : 1); + expect(publishIssueActivity).not.toHaveBeenCalled(); + expect(deps.executeRun).not.toHaveBeenCalled(); + expect(await db.select().from(agentWakeupRequests)).toEqual([]); + expect(await db.select().from(heartbeatRuns)).toHaveLength(kind === "fresh" ? 0 : 1); + }); + it("admits the owner's context compaction maintenance request while its conversation waits", async () => { const f = await waitingCompactionFixture(); const accepted = await f.service.request({ companyId: f.company.id, issue: f.issue, userId: f.user.id }); @@ -287,8 +341,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, }); - f.deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); - f.deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + f.deps.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake); let releaseAvailability!: () => void; const bothAvailable = new Promise(resolve => { releaseAvailability = resolve; }); let arrivals = 0; @@ -333,7 +386,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { const context = enrichWakeContextSnapshot({ contextSnapshot: { ...options.contextSnapshot }, reason: options.reason ?? null, source, triggerDetail, payload: options.payload ?? null }); await db.transaction(async tx => { - await createWakeAdmissionWriter().insertNewDeferredWake(TransactionScope.create(f.company.id, tx), { + await createWakeAdmissionWriter(tx, f.company.id).insertNewDeferredWake({ companyId: f.company.id, agentId, source, triggerDetail, payload: { ...options.payload, manualUserWake: true, [DEFERRED_WAKE_CONTEXT_KEY]: context.contextSnapshot }, requestedByActorType: options.requestedByActorType ?? null, @@ -553,7 +606,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { }).returning(); await db.transaction(async (tx) => { - const store = createFreshIssueRunStore(TransactionScope.create(local.company.id, tx), local.company.id); + const store = createFreshIssueRunStore(tx, local.company.id); expect(await store.saveReceipt({ kind: "promote", wakeId: localDeferredWake.id, agentId: foreignAgent.id, payload: null, })).toBeUndefined(); @@ -889,8 +942,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, }); - deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); - deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + deps.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake); const queue = createHeartbeatQueue(db, deps); if (scenario === "running") { deps.canStartNewExternalBotWork.mockResolvedValue(true); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index ceeb8afa2e7..a3a7cb484d5 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -91,7 +91,6 @@ import { DEFERRED_CHAT_WAKE_RETRY_KEY, assertDurableChatWakeupReceipt, assertDurableChatWakeupRequest, - withDurableChatAdmission, authorizeFailedChatRunRetryWake, FailedChatRunRetryAuthorizationError, unadmittedChatWakeupCondition, @@ -219,7 +218,7 @@ import type { budgetService } from "../budgets.js"; import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js"; import type { instanceSettingsService } from "../instance-settings.js"; import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; -import { bindFreshIssueRunAdmission, hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; +import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; import { crashPoint } from "@paperclipai/shared/crash-points"; import { emitSystemTaskBlockedAfterCommit } from "../product-telemetry.js"; @@ -248,7 +247,7 @@ export interface HeartbeatQueueDependencies extends Pick, "getInvocationBlock">; instanceSettings: Pick, "getExperimental">; runDispatch: Pick, "cancelStaleQueuedRun">; - wakeQueue: Pick, "createAdmissionTransactionScope" | "admitWakeBehindIssueExecution">; + wakeQueue: Pick, "admitIssueWake">; activeRunExecutions: Set; activeRunExecutionPromises: Set>; activeWakeupPromises: Set>; @@ -3521,21 +3520,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } if (issueId) { - // Mention-triggered wakes can request input from another agent, but they must - // still respect the issue execution lock so a second agent cannot start on the - // same issue workspace while the assignee already has a live run. const agentNameKey = normalizeAgentNameKey(agent.name); const cancelledRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = []; const issueActivityPublications: ActivityPublication[] = []; const issuePostCommitActions: IssuePostCommitAction[] = []; - const outcome = await db - .transaction(withDurableChatAdmission(durableRequest, async (tx) => { - await tx.execute( - sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`, - ); - + return wakeQueue.admitIssueWake( + { companyId: agent.companyId, issueId, durableChatRequest: durableRequest }, + async (tx, { admitWakeBehindIssueExecution, admitFreshIssueRun }) => { if (executionWaitRequestId) { const [pending] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), @@ -4730,7 +4723,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende resolvedStrategy, hasResolvablePriorSessionWorkspace, }, - }); + }, issueActivityPublications); return { kind: "skipped" as const }; } } @@ -4802,46 +4795,39 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return { kind: "coalesced" as const, run: activeExecutionRun }; } - const admissionScope = wakeQueue.createAdmissionTransactionScope( - agent.companyId, - tx as unknown as Db, - ); - const admission = await wakeQueue.admitWakeBehindIssueExecution( - admissionScope, - { - companyId: agent.companyId, - issueId: issue.id, - agentId, - agentNameKey, - issueExecutionAgentNameKey: issue.executionAgentNameKey, - activeExecutionRun: { - id: activeExecutionRun.id, - agentId: activeExecutionRun.agentId, - status: activeExecutionRun.status, - contextSnapshot: activeExecutionRun.contextSnapshot, - wakeupRequestId: activeExecutionRun.wakeupRequestId, - }, - allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing, - durableReceipt: receiptRequest - ? { - id: receiptRequest.id, - requestedAt: receiptRequest.requestedAt, - } - : undefined, - reason, - liveRunExecutions, - wakeCommentId, - forceFreshSession: - enrichedContextSnapshot.forceFreshSession === true, - contextSnapshot: enrichedContextSnapshot, - source, - triggerDetail, - payload, - requestedByActorType: opts.requestedByActorType ?? null, - requestedByActorId: opts.requestedByActorId ?? null, - idempotencyKey: opts.idempotencyKey ?? null, + const admission = await admitWakeBehindIssueExecution({ + companyId: agent.companyId, + issueId: issue.id, + agentId, + agentNameKey, + issueExecutionAgentNameKey: issue.executionAgentNameKey, + activeExecutionRun: { + id: activeExecutionRun.id, + agentId: activeExecutionRun.agentId, + status: activeExecutionRun.status, + contextSnapshot: activeExecutionRun.contextSnapshot, + wakeupRequestId: activeExecutionRun.wakeupRequestId, }, - ); + allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing, + durableReceipt: receiptRequest + ? { + id: receiptRequest.id, + requestedAt: receiptRequest.requestedAt, + } + : undefined, + reason, + liveRunExecutions, + wakeCommentId, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, + contextSnapshot: enrichedContextSnapshot, + source, + triggerDetail, + payload, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }); if (admission.kind === "coalesced") { return { @@ -5075,7 +5061,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }, issueActivityPublications); } - const admitFreshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db); const newRun = await admitFreshIssueRun({ receipt: { agentId, @@ -5130,61 +5115,33 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // transitions to "running" — Fix A (lazy locking). return { kind: "queued" as const, run: newRun }; - })) - .catch((error) => { - if (isExternalChatWaitAuthorizationContention(error)) - return { kind: "deferred" as const }; - throw error; - }); - - // The admission has committed. What follows is dispatch of saved rows. - if (outcome.kind !== "deferred" && outcome.kind !== "skipped") admission?.saved(); - for (const publication of issueActivityPublications) publishActivity(publication); - await executeIssuePostCommitActions(db, issuePostCommitActions); - - // Telemetry for the cancelled runs is best-effort background work. - // Fire it here and never await it: none of the lifecycle work below, - // nor this function's return, depends on it, so a slow telemetry - // lookup must not delay them. - for (const cancelledRun of cancelledRunsToEmit) { - void emitAgentTaskRun(db, cancelledRun); - } - - if (outcome.kind === "durable") { - return outcome.receipt.runId ? getRun(outcome.receipt.runId) : null; - } - if (outcome.kind === "deferred" || outcome.kind === "skipped") { - return null; - } - if (outcome.kind === "coalesced") { - await attachQueuedSetupRequest(agent); - await startNextQueuedRunForAgent(agent.id); - return outcome.run; - } - if (outcome.kind === "replayed") { - if (outcome.run.status === "queued") { - await attachQueuedSetupRequest(agent); - await startNextQueuedRunForAgent(agent.id); - } - return outcome.run; - } - - const newRun = outcome.run; - publishLiveEvent({ - companyId: newRun.companyId, - type: "heartbeat.run.queued", - payload: { - runId: newRun.id, - agentId: newRun.agentId, - invocationSource: newRun.invocationSource, - triggerDetail: newRun.triggerDetail, - wakeupRequestId: newRun.wakeupRequestId, }, - }); - - await attachQueuedSetupRequest(agent); - await startNextQueuedRunForAgent(agent.id); - return newRun; + { + saved: () => admission?.saved(), + publishIssueActivity() { + for (const publication of issueActivityPublications) publishActivity(publication); + }, + executeIssueActions: () => executeIssuePostCommitActions(db, issuePostCommitActions), + cancelledRuns: cancelledRunsToEmit, + emitCancelledRun: (run) => emitAgentTaskRun(db, run), + readRun: getRun, + publishQueuedRun(run) { + publishLiveEvent({ + companyId: run.companyId, + type: "heartbeat.run.queued", + payload: { + runId: run.id, + agentId: run.agentId, + invocationSource: run.invocationSource, + triggerDetail: run.triggerDetail, + wakeupRequestId: run.wakeupRequestId, + }, + }); + }, + attachSetupRequest: () => attachQueuedSetupRequest(agent), + dispatch: () => startNextQueuedRunForAgent(agent.id), + }, + ); } if (durableRequest) throw new Error("chat_inbound_wakeup_binding_denied"); diff --git a/server/src/services/native-runtime/native-remote-retry-admission.test.ts b/server/src/services/native-runtime/native-remote-retry-admission.test.ts index 0a67db1d02f..d5c48da79b3 100644 --- a/server/src/services/native-runtime/native-remote-retry-admission.test.ts +++ b/server/src/services/native-runtime/native-remote-retry-admission.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { resolve } from "node:path"; import { and, eq, sql } from "drizzle-orm"; -import { agents, companies, completionContracts, databaseWorkPending, environmentLeases, reconcileDatabaseWork, heartbeatRunEvents, heartbeatRuns, issues, issueRelations, +import { agents, agentWakeupRequests, companies, completionContracts, databaseWorkPending, environmentLeases, reconcileDatabaseWork, heartbeatRunEvents, heartbeatRuns, issues, issueRelations, issueTreeHolds, issueTreeHoldMembers, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, nativeRunnerCheckpoints } from "@paperclipai/db"; import { expect, it, vi } from "vitest"; import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; @@ -156,11 +156,12 @@ describeEmbeddedPostgres("remote native retry execution admission", () => { const dependencies = createHeartbeatQueueTestDependencies(f.db); dependencies.getAgent.mockResolvedValue(agent); dependencies.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(f.db).getExperimental()); - dependencies.wakeQueue.admitWakeBehindIssueExecution.mockResolvedValue({ kind: "deferred" }); - await createHeartbeatQueue(f.db, dependencies).enqueueWakeup(f.agentId, { source: "assignment", reason: "issue_assigned", - requestedByActorType: "system", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } }); + expect(await createHeartbeatQueue(f.db, dependencies).enqueueWakeup(f.agentId, { source: "assignment", reason: "issue_assigned", + allowRunCoalescing: false, requestedByActorType: "system", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } })).toBeNull(); expect(dependencies.executeRun).not.toHaveBeenCalled(); - expect(dependencies.wakeQueue.admitWakeBehindIssueExecution).toHaveBeenCalledOnce(); + const [wake] = await f.db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${f.issueId}`)); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) .toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); }); diff --git a/server/src/services/workspace-recovery-admission.test.ts b/server/src/services/workspace-recovery-admission.test.ts index b5bccc7cb05..56534a0f4dd 100644 --- a/server/src/services/workspace-recovery-admission.test.ts +++ b/server/src/services/workspace-recovery-admission.test.ts @@ -1,13 +1,12 @@ -import { createAdmissionTransactionScope, createWakeAdmissionReader, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; -import { createAdmitWakeBehindIssueExecution } from "../modules/wake-queue/application/use-cases.js"; -import { filterZombieCoalesceTarget, heartbeatService } from "./heartbeat.js"; +import { createWakeAdmissionReader, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { heartbeatService } from "./heartbeat.js"; import { mergeCoalescedContextSnapshot } from "./heartbeat/run-preparation.js"; import { recoveryService } from "./recovery/index.js"; import { inspectWorkspaceSource } from "./workspace-source-inspection.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; import { heartbeatRunEvents, agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations, plugins } from "@paperclipai/db"; -import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "./heartbeat/queue.js"; +import { createHeartbeatQueue } from "./heartbeat/queue.js"; import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js"; import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js"; import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js"; @@ -1425,11 +1424,11 @@ describe("automatic retained workspace recovery", () => { const incoming = { source: "issue.comment", issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: followup.id, wakeCommentIds: [followup.id] }; await db.transaction(async tx => { - const scope = createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db); + const reader = createWakeAdmissionReader(tx, f.companyId); const actor = { companyId: f.companyId, wakeupRequestId: wake.id, requestedByActorType: "user", requestedByActorId: "operator" }; - expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, actor)).toBe(true); - expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, { ...actor, requestedByActorId: "other-user" })).toBe(false); - await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(scope, { + expect(await reader.matchesActiveWakeActor(actor)).toBe(true); + expect(await reader.matchesActiveWakeActor({ ...actor, requestedByActorId: "other-user" })).toBe(false); + await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({ companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, activeExecutionRunStatus: queued.status, mergedContextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot, incoming, { preserveExistingInteractionContinuation: true }), @@ -1454,12 +1453,15 @@ describe("automatic retained workspace recovery", () => { await db.update(agentWakeupRequests).set({ runId: null }).where(eq(agentWakeupRequests.id, wake.id)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)); await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); - const reader = createWakeAdmissionReader(), sameActor = vi.spyOn(reader, "matchesActiveWakeActor"); - queueDeps.wakeQueue.createAdmissionTransactionScope.mockImplementation(createAdmissionTransactionScope); - queueDeps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(createAdmitWakeBehindIssueExecution({ - reader, writer: createWakeAdmissionWriter(), helpers: { filterZombieCoalesceTarget, mergeCoalescedContextSnapshot, - shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, - })); + const admitIssueWake = queueDeps.wakeQueue.admitIssueWake.getMockImplementation()!; + queueDeps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => + admitIssueWake(target, (tx, admission) => evaluate(tx, { + ...admission, + admitWakeBehindIssueExecution(input) { + expect(input.durableReceipt).toBeUndefined(); + return admission.admitWakeBehindIssueExecution(input); + }, + }), effects)); queueDeps.liveRunExecutions.has.mockImplementation(id => t.deps.activeRunExecutions.has(id)); const stopped = await run(f.runId); if (kind !== "active") { @@ -1511,8 +1513,6 @@ describe("automatic retained workspace recovery", () => { const current = await run(admitted!.id); const [claimedWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, current.wakeupRequestId!)); expect(current.contextSnapshot?.wakeCommentIds).toEqual(ids); - expect(sameActor).not.toHaveBeenCalled(); - expect(queueDeps.wakeQueue.admitWakeBehindIssueExecution.mock.calls.every(([, input]) => input.durableReceipt === undefined)).toBe(true); expect(queueDeps.executeRun).not.toHaveBeenCalled(); return { ...t, queued: current, wake: claimedWake, followup, expectedIds: ids, latestAuthor: kind === "deferred-different" ? "third-user" : kind === "active" ? "second-user" : "operator" }; } @@ -1887,10 +1887,10 @@ describe("automatic retained workspace recovery", () => { }, 30_000); it("preserves ordinary no-issue deferred merge semantics", async () => { - const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter(); + const f = await seed(), original = await run(f.runId); const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId, source: "automation", status: "deferred_issue_execution", payload: { text: "original" }, requestedByActorType: "system" }).returning(); - await db.transaction(tx => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { + await db.transaction(tx => createWakeAdmissionWriter(tx, f.companyId).mergeIntoExistingDeferredWake({ companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { text: "merged" }, nextCoalescedCount: 1, coalescedReceipt: { agentId: original.agentId, source: "automation", triggerDetail: null, reason: null, payload: { text: "saved" }, requestedByActorType: "system", requestedByActorId: null, idempotencyKey: null, runId: null }, })); @@ -1901,12 +1901,12 @@ describe("automatic retained workspace recovery", () => { }); it("rolls back saved primary and carrier overwrite if incoming mixed-author receipt persistence fails", async () => { - const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter(); + const f = await seed(), original = await run(f.runId); const [first] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "operator", body: "First saved input" }).returning(); const [second] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "second-user", body: "Second saved input" }).returning(); const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId, source: "automation", status: "deferred_issue_execution", payload: { issueId: f.issueId, commentId: first.id }, requestedByActorType: "user", requestedByActorId: "operator" }).returning(); - const merge = (tx: Parameters[0]>[0], id?: string) => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { + const merge = (tx: Parameters[0]>[0], id?: string) => createWakeAdmissionWriter(tx, f.companyId).mergeIntoExistingDeferredWake({ companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { issueId: f.issueId, commentId: second.id }, nextCoalescedCount: 1, coalescedReceipt: { ...(id ? { id } : {}), agentId: original.agentId, source: "automation", triggerDetail: null, reason: "issue_commented", payload: { issueId: f.issueId, commentId: second.id, coalescedIntoWakeupRequestId: "untrusted" }, requestedByActorType: "user", requestedByActorId: "second-user", idempotencyKey: null, runId: null }, });