From 0ca4b7c4c06196a3e6bf911c928ce346f96646ec Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:59:42 -0500 Subject: [PATCH 1/2] refactor(heartbeat): extract run cancellation and cleanup (#15685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The heartbeat service admits, executes, and settles agent runs. > - Earlier extractions separated workspace, preparation, state, retry, recovery, queue, and lifecycle handling. > - Cancellation and resource release still live inside the main service. > - These operations share execution ownership and saved-comment continuation rules. > - This pull request moves them into one run-control module and adds boundary tests. > - Reviewers can inspect cancellation and cleanup separately from adapter execution. ## Linked Issues or Issue Description **What existing behavior does this improve?** The structure and testability of heartbeat cancellation and cleanup. **Subsystem affected** server/ — orchestration services. **Current behavior** Stop, pause and budget cancellation, environment lease release, and saved-comment delivery live in `heartbeat.ts`. **Proposed behavior** Keep these operations in `server/src/services/heartbeat/run-control.ts`. Bind their database and existing callbacks with `createHeartbeatRunControl`. **Reason and benefit** This removes 1,115 lines from `heartbeat.ts`, leaving 10,443 lines. The new module has 1,360 lines. Related cancellation and cleanup rules stay together in one file. **Breaking changes** None. Function bodies, service methods, public helpers, status conditions, and side effects keep their behavior. **Additional context** Continues the merged lifecycle extraction in #15672. Related cancellation work: #15212. Searches of open and closed issues and PRs found no duplicate run-control extraction. This does not add a roadmap feature. ## What Changed - Move run cancellation, pause and budget cancellation, lease release, issue-lock release, and saved-comment resumption into `heartbeat/run-control.ts`. - Supply lifecycle, queue, recovery, and cleanup callbacks through an explicit dependency interface. - Keep executor and cancellation maps shared by all service instances. Preserve callback construction order with forwarding functions. - Keep the existing helpers available from `heartbeat.ts`. - Add 15 tests for inert construction, cancellation gates, shared Stop barriers, failed termination, warm-resource retention, pending wake batches, project scope, and stale budget enforcement. - Document the module boundary in `doc/DEVELOPING.md`. ## Verification - Before extraction: **298 tests passed across six existing cancellation and cleanup suites**. - After extraction: **313 tests passed across seven files**, including the 15 new module tests and real PostgreSQL coverage. ```sh pnpm exec vitest run server/src/services/heartbeat/run-control.test.ts server/src/__tests__/heartbeat-native-runner-cancellation.test.ts server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts server/src/services/run-cancellation.test.ts server/src/services/adapter-execution-control.test.ts server/src/services/explicit-native-continuation.test.ts server/src/__tests__/native-cancellation-request.integration.test.ts ``` - Wider queue, recovery, reassignment, comment delivery, and native cancellation verification: **139 tests passed across nine files**, including the final module tests. The recovery suite first hit its 20-second database setup timeout while local builds were running. After removing verified orphaned PostgreSQL headers, its isolated run passed **380 of 381 cases**. One teardown assertion exceeded its one-second wait for a successor to settle; that exact case passed in a filtered rerun. Combined focused coverage is **818 distinct passing tests across 16 files**. The module suite appears in both runs. ```sh pnpm exec vitest run server/src/services/heartbeat/run-control.test.ts server/src/__tests__/heartbeat-comment-wake-batching.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-native-cleanup-admission.test.ts server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts server/src/services/heartbeat/queue.test.ts server/src/services/heartbeat/recovery.test.ts server/src/services/heartbeat/retries.test.ts server/src/services/heartbeat-stop-metadata.test.ts server/src/services/native-runtime/native-cancellation-request.test.ts ``` ```sh pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "does not adopt unrelated queued comments for a non-coalescing recipient after Stop" ``` - Full `pnpm -r typecheck` and `pnpm build` passed on head `b26efd5f9de390ef8a675f84980f49f7fb7799c2`. - Full local `pnpm test:run` was started on this head and stopped after the complete CI suite passed. It did not finish and is not counted as a full local pass. - All **54 final-head checks passed**; two optional Storybook checks were skipped. This includes the full general and serialized test matrix, browser tests, typechecks, build, Runner verification, and canary dry run. The complete recovery suite passed in CI. [CI run](https://github.com/paperclipai/paperclip/actions/runs/37945389261). - Greptile completed on head `b26efd5f9de390ef8a675f84980f49f7fb7799c2` with **5/5 and no actionable findings**. There are no review threads or merge conflicts. - Structural comparison confirms all 21 moved function bodies, the cancellation options type, remaining service logic, and all 140 existing public exports are preserved. ## Risks Moving closures can break callback construction order or shared Stop barriers. The factory starts no work and receives the same process-wide maps and existing callbacks. Tests cover concurrent Stop callers and failed termination. Native authority, status predicates, provider receipts, cleanup gates, and budget enforcement keep their original bodies. This PR has no schema or API changes. ## Model Used OpenAI GPT-6 via Codex. The exact serving model ID and context window were not exposed in this session. Used reasoning, repository inspection, shell tools, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/DEVELOPING.md | 10 + server/src/services/heartbeat.ts | 1303 ++-------------- .../services/heartbeat/run-control.test.ts | 310 ++++ server/src/services/heartbeat/run-control.ts | 1360 +++++++++++++++++ 4 files changed, 1774 insertions(+), 1209 deletions(-) create mode 100644 server/src/services/heartbeat/run-control.test.ts create mode 100644 server/src/services/heartbeat/run-control.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index f6fcad77477..35fcb16bc1a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1290,6 +1290,16 @@ reporting and forwarding callbacks preserve construction order for the other heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`. Keep lifecycle policy changes separate from this extraction and adapter execution. +Run cancellation and cleanup are in `server/src/services/heartbeat/run-control.ts`. +It owns Stop, agent and budget cancellation, environment lease release, and +resumption of saved comments after execution stops. `createHeartbeatRunControl` +binds the database, lifecycle and admission callbacks, and shared executor +barriers without starting work. The service retains the process-wide execution +and cancellation maps so scheduler and route instances observe the same owners. +Forwarding callbacks preserve construction order for budget, retry, recovery, +and queue services. Existing public helpers remain re-exported by `heartbeat.ts`. +Keep cancellation policy changes separate from this extraction and execution. + ## Wake Context Delivery Built-in adapters deliver wake context through the run prompt, including structured diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index be77bf2f401..616cdcdeabf 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,15 @@ +import { + cancelHeartbeatNativeRun, + terminateHeartbeatRunProcess, + providerResourceDispositionForTerminalRun, + createHeartbeatRunControl, + type CancelRunOptions, +} from "./heartbeat/run-control.js"; +export { + leaseReleaseStatusForRunStatus, + providerResourceDispositionForTerminalRun, + cancelHeartbeatNativeRun, +} from "./heartbeat/run-control.js"; import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus, @@ -25,7 +37,6 @@ export { } from "./heartbeat/queue.js"; import { isProcessAlive, - CANCELLABLE_HEARTBEAT_RUN_STATUSES, createHeartbeatRecovery, } from "./heartbeat/recovery.js"; import { @@ -247,10 +258,8 @@ import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/as import { externalObjectService } from "./external-objects.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { dotRunnerBroker } from "./dot-runner-broker.js"; -import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; + import { - nativeRetryCancellationCommitCondition, - claimCancellationRequest, startupCancellationFence, } from "./native-runtime/native-cancellation-request.js"; import { @@ -262,7 +271,10 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./age import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js"; -import { cancellationResultJson, canContinueCancelledRun, readRunCancellation, requestedRunCancellation } from "./run-cancellation.js"; +import { + cancellationResultJson, + readRunCancellation, +} from "./run-cancellation.js"; import { withNativeWorkspaceFinalizationOwnership, NativeWorkspaceFinalizationBusyError, @@ -278,11 +290,8 @@ import { hasWorkspaceRestoreFailure, } from "@paperclipai/shared"; -import { settleSlackConversation } from "./slack-conversation-lifecycle.js"; import { toolActionDeliveryService } from "./tool-action-delivery.js"; - -import { readQueuedInteractionResponse } from "./queued-interaction-response.js"; import { isConversation, prepareConversationTurn, @@ -293,29 +302,18 @@ import { getConversationConfirmationContext } from "./conversation-confirmation- import { recordNativeLocalProcessStop, } from "./native-local-process-stop.js"; -import { - isAcknowledgedNativeStop, - acknowledgedNativeStopExecutionHasStopped, -} from "./acknowledged-native-stop.js"; + import { legacyControllerBootId, renewLegacyControllerLease, watchLegacyControllerLease, } from "./legacy-controller-lease.js"; -import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; + import { - hasRemoteTerminationReceipt, remoteExecutionHasStopped, - stoppedRemoteCleanupScopes, } from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; -import { - admitExplicitNativeContinuation, - undeliveredLegacyUserCommentIds, -} from "./explicit-native-continuation.js"; -import { - isCancelledNativeStartup, -} from "./cancelled-native-startup.js"; + import { connectionIntentService } from "./connection-intents.js"; import { prepareManagedAiRuntime, @@ -327,12 +325,11 @@ import { import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared"; import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js"; import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js"; -import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js"; import { - CONVERSATION_CONTINUATION_POLICY, + getExecutionBlocker, +} from "./execution-blocker.js"; +import { claimedAdapterType, - runUsedConversationAdapter, - hasConversationContinuationPolicy, } from "./conversation-continuation.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; @@ -345,10 +342,8 @@ import { } from "./legacy-execution-recovery.js"; import { adapterExecutionControls, - captureAdapterStopOwnership, createAdapterExecutionControl, registerAdapterExecutionControl, - waitForAdapterStop, } from "./adapter-execution-control.js"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; @@ -386,14 +381,11 @@ import { eq, exists, getTableColumns, - gt, gte, inArray, isNull, - isNotNull, lt, lte, - ne, notInArray, or, sql, @@ -404,7 +396,6 @@ import { ISSUE_DISPOSITION_REPAIR_RETRY_REASON, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, isEnvironmentDriverSupportedForAdapter, - type EnvironmentLeaseStatus, type ExecutionWorkspace, type IssueExecutionMonitorClearReason, type IssueExecutionMonitorPolicy, @@ -419,11 +410,9 @@ import { chatActions, companies, completionContracts, - environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, - issueRecoveryActions, issues, nativeRunFinalizations, projects, @@ -449,10 +438,7 @@ export { scrubGitCredentialText }; import { publishLiveEvent } from "./live-events.js"; import { - queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, - withQueuedCommentIdsInWakePayload, - withQueuedCommentIdsInRunContext, } from "./issue-queued-comment-queue.js"; import { documentService } from "./documents.js"; import { getTaskPlanContext } from "./task-plan-context.js"; @@ -463,10 +449,8 @@ import { buildNativeExecutionInput, buildNativeExecutionWithCheckpoint, buildNativeRuntimeContext, - cancelNativeSession, claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, - closeWarmNativeSessionsForEnvironment, reserveWarmNativeInstructionDirectory, dispatchNativeSessionResumptions, ensureNativeCompletionContract, @@ -560,7 +544,9 @@ import { reportRunFailure } from "./run-failure-report.js"; import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; -import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js"; +import { + budgetService, +} from "./budgets.js"; import { secretService, } from "./secrets.js"; @@ -647,7 +633,6 @@ import { } from "./workspace-operations.js"; import { isProcessGroupAlive, - terminateLocalService, } from "./local-service-supervisor.js"; import { isRuntimeOwnedGitBranch, @@ -709,7 +694,6 @@ import { } from "../modules/run-dispatch/index.js"; import { createWakeQueue, - WakeQueueApplicationError, type IssueSnapshot as WakeQueueIssueSnapshot, type PostCommitEffect as WakeQueuePostCommitEffect, type ReleaseRecoveryBlockedNoticeKind, @@ -1018,21 +1002,6 @@ export function getTaskDrainStatus(): { }; } -export function leaseReleaseStatusForRunStatus( - status: string | null | undefined, -): Extract { - if (status === "cancelled") return "expired"; - return status === "failed" || status === "timed_out" ? "failed" : "released"; -} - -export function providerResourceDispositionForTerminalRun( - desired: ProviderResourceDisposition | undefined, - status: string | null | undefined, -): ProviderResourceDisposition | undefined { - if (desired !== "keep_running") return desired; - return status === "succeeded" ? desired : "stop_and_retain"; -} - export interface NativeSandboxLifecycle { runnerProcess: "per_turn" | "warm"; sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; @@ -1283,33 +1252,6 @@ function isSameTaskScope(left: string | null, right: string | null) { return (left ?? null) === (right ?? null); } -async function terminateHeartbeatRunProcess(input: { - pid: number | null | undefined; - processGroupId: number | null | undefined; - graceMs?: number; - signal?: NodeJS.Signals; -}) { - const pid = input.pid ?? null; - const processGroupId = input.processGroupId ?? null; - if (typeof pid !== "number" && typeof processGroupId !== "number") return; - - await terminateLocalService( - { - pid: - typeof pid === "number" && Number.isInteger(pid) && pid > 0 - ? pid - : (processGroupId ?? 0), - processGroupId: - typeof processGroupId === "number" && - Number.isInteger(processGroupId) && - processGroupId > 0 - ? processGroupId - : null, - }, - { forceAfterMs: input.graceMs, signal: input.signal }, - ); -} - export type HeartbeatEnvironmentRuntime = ReturnType< typeof environmentRuntimeService >; @@ -1351,38 +1293,6 @@ export interface HeartbeatServiceOptions { }) => Promise; } -export async function cancelHeartbeatNativeRun(input: { - db: Db; - runId: string; - reason: string; - runtimeMode: string | null; - cancellationRequestId?: string; - cancel?: ( - runId: string, - reason: string, - options: { db: Db; scope: "run"; cancellationRequestId?: string }, - ) => Promise<{ decision: unknown | null; auditId: string | null }>; -}) { - if (input.runtimeMode !== "native") { - return { decision: null, auditId: null }; - } - const cancellation = input.cancel - ? await input.cancel(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }) - : await cancelNativeSession(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }); - if (!cancellation.decision || !cancellation.auditId) { - throw new Error("native_cancellation_outcome_not_audited"); - } - return cancellation; -} - class NativeSessionResumeScheduledError extends Error { constructor(readonly original: unknown) { super("Native session recovery has been scheduled for the same run."); @@ -2061,6 +1971,74 @@ export function heartbeatService( return runLifecycle.publishRunLifecyclePluginEvent(...args); } + const runControl = createHeartbeatRunControl(db, { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + }); + + const { + cancelInvocationsForAgentsInternal, + } = runControl; + + // Earlier factories capture these callbacks before run-control construction. + function cancelBudgetScopeWork(...args: Parameters) { + return runControl.cancelBudgetScopeWork(...args); + } + + function releaseIssueExecutionAndPromote(...args: Parameters) { + return runControl.releaseIssueExecutionAndPromote(...args); + } + + function releaseEnvironmentLeasesForRun(...args: Parameters) { + return runControl.releaseEnvironmentLeasesForRun(...args); + } + + function acknowledgeRemoteStop(...args: Parameters) { + return runControl.acknowledgeRemoteStop(...args); + } + + function resumeRemoteStopComments(...args: Parameters) { + return runControl.resumeRemoteStopComments(...args); + } + + function cancelRunInternal(...args: Parameters) { + return runControl.cancelRunInternal(...args); + } + + function resumeExecutionWaitComments(...args: Parameters) { + return runControl.resumeExecutionWaitComments(...args); + } + + function resumeQueuedCommentInterrupt(...args: Parameters) { + return runControl.resumeQueuedCommentInterrupt(...args); + } + + function resumeSavedLegacyComments(...args: Parameters) { + return runControl.resumeSavedLegacyComments(...args); + } + + function cancelActiveForAgentInternal(...args: Parameters) { + return runControl.cancelActiveForAgentInternal(...args); + } + // Applies the post-commit effects a wake-queue release returns, exactly as // the original release function did before its writes moved into that // module: publish + dispatch a promoted or recovery run, log a reopened @@ -2123,422 +2101,6 @@ export function heartbeatService( } const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup }); - async function releaseEnvironmentLeasesForRun(input: { - runId: string; - companyId: string; - agentId: string; - status: string | null | undefined; - failureReason?: string | null; - providerResourceDisposition?: ProviderResourceDisposition; - nativeLifecycleTelemetry?: { - provider: string; - harness: string; - lifecycleMode: "per_turn" | "warm"; - sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; - }; - }) { - const leaseOwnerRun = await getRun(input.runId); - if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; - // Recovery can finish workspace copy-back outside the executor's finally. - // Successful copy-back does not earn warm retention for a failed turn. - const status = leaseOwnerRun?.status ?? input.status; - const providerResourceDisposition = providerResourceDispositionForTerminalRun( - input.providerResourceDisposition, - status, - ); - if (providerResourceDisposition === "destroy") { - const closeResult = await ( - options.closeWarmNativeSessionsForRun ?? - (async ({ runId, reason }) => { - const environmentIds = await db - .selectDistinct({ environmentId: environmentLeases.environmentId }) - .from(environmentLeases) - .where( - and( - eq(environmentLeases.heartbeatRunId, runId), - eq(environmentLeases.status, "active"), - ), - ) - .then((rows) => - rows.flatMap((row) => - typeof row.environmentId === "string" && - row.environmentId.length > 0 - ? [row.environmentId] - : [], - ), - ); - const aggregate = { closed: 0, busy: 0, failed: 0 }; - for (const environmentId of environmentIds) { - const result = await closeWarmNativeSessionsForEnvironment({ - environmentId, - reason, - }); - aggregate.closed += result.closed; - aggregate.busy += result.busy; - aggregate.failed += result.failed; - } - return aggregate; - }) - )({ - runId: input.runId, - reason: "terminal heartbeat run destroyed its environment lease", - }).catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to close warm native sessions before environment lease destruction", - ); - return { closed: 0, busy: 0, failed: 1 }; - }); - // A failed remote checkpoint cannot veto destruction of the isolated - // sandbox after the run stopped. Provider destruction supplies the exit - // proof; it does not turn the interrupted checkpoint into a success. - const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && - ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) - ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( - eq(environmentLeases.companyId, input.companyId), - eq(environmentLeases.heartbeatRunId, input.runId), - )) - : []; - const canDestroyRemote = remoteLeases.length > 0 && - remoteLeases.every(lease => lease.provider && lease.provider !== "local"); - if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { - logger.warn( - { runId: input.runId, warmNativeSessions: closeResult }, - "deferred environment lease destruction until warm native sessions close", - ); - return; - } - } - const releaseResult = await envOrchestrator - .releaseForRun({ - heartbeatRunId: input.runId, - companyId: input.companyId, - agentId: input.agentId, - status: leaseReleaseStatusForRunStatus(status), - failureReason: input.failureReason ?? undefined, - providerResourceDisposition, - nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, - }) - .catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to release environment leases for heartbeat run", - ); - return null; - }); - for (const releaseError of releaseResult?.errors ?? []) { - logger.warn( - { - err: releaseError.error, - leaseId: releaseError.leaseId, - runId: input.runId, - }, - "failed to release environment lease for heartbeat run", - ); - } - await acknowledgeRemoteStop(input.runId, input.companyId); - } - - async function acknowledgeRemoteStop(runId: string, companyId: string) { - // The provider receipt arrives after adapter settlement. A remote ACP child - // has no host PID, so only this target-aware boundary can acknowledge Stop. - const stopped = await getRun(runId); - if (stopped?.runtimeMode === "native") { - const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); - for (const remoteCleanupScope of scopes ?? []) { - completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); - } - } - if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && - parseObject(stopped.resultJson?.executionCancellation).state === "requested" && - await runUsedConversationAdapter(db, stopped) && - await remoteExecutionHasStopped(db, companyId, runId)) { - await db.update(heartbeatRuns).set({ - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ - executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), - state: "acknowledged", acknowledgedAt: new Date().toISOString(), - proof: "provider_termination_receipt" }, - conversationContinuation: CONVERSATION_CONTINUATION_POLICY, - })}::jsonb`, - updatedAt: new Date(), - }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.status, "cancelled"))); - } - } - - async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { - if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; - const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt - ? await db.select().from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - const cancelledPreparation = run.runtimeMode === "legacy" && - await isCancelledNativeStartup(db, run, preparationCoordinator); - if (run.runtimeMode !== "native" && !cancelledPreparation && - parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && - !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; - const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); - if (!issueId) return; - const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; - const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, - leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - // Stop ends one response. Saved user input can enter ordinary admission - // once its executor settles; it does not need a manufactured crash incident. - // The ordinary path still enforces task holds, ownership, and native session - // cleanup before starting a provider. - const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && - !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && - ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && - await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - const legacyContinuation = run.runtimeMode === "legacy" && - hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; - const pending = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - requestId ? eq(agentWakeupRequests.id, requestId) : undefined, - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); - for (const wake of pending) { - if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; - let payload = parseObject(wake.payload); - if (payload.queuedCommentInterrupt) { - await resumeQueuedCommentInterrupt(wake.companyId, wake.id); - continue; - } - let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); - let commentId = deriveCommentId(context, payload); - let requestedByActorId = wake.requestedByActorId; - const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; - if (run.runtimeMode === "native" || cancelledPreparation) { - const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, - queuedCommentIdsFromWakePayload(payload)); - if (!ids.length) continue; - payload = withQueuedCommentIdsInWakePayload(payload, ids); - context = withQueuedCommentIdsInRunContext(context, ids); - commentId = ids.at(-1)!; - // A coalesced queue can contain several authors. Its saved comments, - // not the outer wake's first author, authorize the remaining input. - const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), - )); - requestedByActorId = author?.id ?? null; - } - if (legacyContinuation || stoppedNativeContinuation) { - if (!commentId || !run.finishedAt || !requestedByActorId || - !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), - sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), - eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), - isNull(issueComments.createdByRunId), - stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), - )); - if (!comment?.body.trim()) continue; - } else { - let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; - const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, - agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, - reason, commentId, successorRunId: randomUUID(), dryRun: true, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - onBlocked: (reason, message) => { wait = { reason, message }; }, - }); - if (!admitted) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); - continue; - } - } - // Re-enter ordinary admission with the original user's authority. It - // atomically adopts the deferred comments and still applies every gate. - await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], - reason, payload, contextSnapshot: context, - requestedByActorType: "user", requestedByActorId, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); - break; - } - } - - async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { - return resumeSavedLegacyComments(companyId, queueId, true, opts); - } - - async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { - const [wake] = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - )); - if (!wake) return; - const payload = parseObject(wake.payload); - let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); - let commentIds = queuedCommentIdsFromWakePayload(payload); - const issueId = readNonEmptyString(payload.issueId); - if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; - const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); - if (!commentIds.length && !response) return; - // Resolved cards are immutable input. Only an explicit Interrupt click can - // authorize continuation across a stopped execution; ordinary completion - // uses normal deferred-wake promotion. - if (response && !interrupted) return; - if (!interrupted) { - commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); - if (!commentIds.length) return; - // The queue itself may have begun as a system wake. The saved human - // comment, not that wake's origin or mutable caller payload, is authority. - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), - eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), - isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), - )).orderBy(desc(issueComments.createdAt)).limit(1); - if (!comment?.body.trim() || !comment.authorUserId) return; - actorId = comment.authorUserId; - } - if (!actorId) return; - const agent = await getAgent(wake.agentId); - if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; - const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (active) return; - if (interrupted && opts?.retryCleanup) { - // Only the HTTP click grants an extra cleanup attempt. Periodic retries - // reuse the intent to deliver, never a fresh provider teardown budget. - const sourceRun = await db.transaction(async tx => { - const [task] = await tx.select().from(issues).where(and( - eq(issues.companyId, companyId), eq(issues.id, issueId), - )).for("update"); - const [current] = await tx.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, - )); - if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || - !current || (!queuedCommentIdsFromWakePayload(current.payload).length && - !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; - const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (successor) return null; - const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); - const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), - eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), - inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - )).then(rows => rows[0]) : null; - if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; - // Older ephemeral leases recorded successful cleanup without a provider - // receipt. Re-verify them through the recorded teardown path; a timestamp - // alone never certifies termination. Retained/reusable resources stay put. - const historical = await tx.select().from(environmentLeases).where(and( - eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), - eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), - inArray(environmentLeases.status, ["released", "expired", "failed"]), - )).for("update"); - for (const lease of historical) { - if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; - // Provider resource IDs identify physical sandboxes. A lease in any - // company can still own this resource; never destroy it on behalf of - // this company. This existence-only guard exposes no foreign data. - const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( - ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), - eq(environmentLeases.providerLeaseId, lease.providerLeaseId), - or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), - )).limit(1); - if (otherOwner) continue; - await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) - .where(eq(environmentLeases.id, lease.id)); - } - return run; - }); - if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { - companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", - } }); - } - const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); - delete deliveryPayload.queuedCommentInterrupt; - await enqueueWakeup(wake.agentId, { - source: "on_demand", triggerDetail: "manual", reason: "issue_commented", - payload: deliveryPayload, contextSnapshot: response - ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, - responsibleUserId: actorId } - : withQueuedCommentIdsInRunContext({ - issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, - }, commentIds), - requestedByActorType: "user", requestedByActorId: actorId, - ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), - issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, - idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, - }, queueId); - } - - async function resumeExecutionWaitComments() { - if ((await getSchedulingSuppression()).suppressed) return; - const waits = await db.select({ wake: agentWakeupRequests }) - .from(agentWakeupRequests) - .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), - sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, - eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) - .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), - executionBlockerPredicate(), - ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - sql`${agentWakeupRequests.payload}->'executionWait' is not null`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - notInArray(issues.status, ["done", "cancelled"]))) - .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); - const seen = new Set(); - for (const { wake } of waits) { - const issueId = String(wake.payload?.issueId); - if (seen.has(issueId)) continue; - seen.add(issueId); - // Advance the cursor even for invalid evidence so one damaged task cannot - // starve later requests in this bounded scan. Preserve concurrent edits. - const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( - eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - )).returning({ id: agentWakeupRequests.id }); - if (!claimed) continue; - // Match normal admission's deterministic current blocker selection. An - // arbitrary historical action must not choose the retry's source run. - const blocker = await getExecutionBlocker(db, wake.companyId, issueId); - const sourceId = blocker?.runId; - if (!sourceId || !isUuidLike(sourceId)) continue; - const run = await getRun(sourceId, { includeExecutionEvidence: true }); - if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; - if (canContinueCancelledRun(run)) { - await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); - }); - continue; - } - await resumeRemoteStopComments(run, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); - }); - } - } - async function getAgentInvokability( agent: typeof agents.$inferSelect | null | undefined, ) { @@ -10618,683 +10180,6 @@ export function heartbeatService( } } - async function releaseIssueExecutionAndPromote( - run: Pick, - options: { suppressImmediateRecovery?: boolean } = {}, - ) { - try { - const source = await getRun(run.id); - const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ - companyId: run.companyId, - runId: run.id, - now: new Date(), - // A durable authentication card owns recovery. This covers the review path, - // while continuation classification blocks periodic generic retries. - suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), - }); - await applyWakeQueuePostCommitEffects(postCommitEffects); - const completed = await getRun(run.id); - const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) - ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; - if (completed?.status === "succeeded" && issueId) { - await settleSlackConversation(db, run.companyId, issueId).catch((err) => { - logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); - }); - } - } catch (error) { - if ( - error instanceof WakeQueueApplicationError && - error.code === "responsible_user_unresolved" - ) { - // Every other `responsible_user_unresolved` HttpError in this file - // carries `code` inside its own `details`; match that shape here too. - throw new HttpError(422, error.message, { code: error.code, ...error.details }); - } - throw error; - } - } - - async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { - const runIssueId = sql< - string | null - >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await db - .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) - .from(heartbeatRuns) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${runIssueId}`, - ), - ) - .where( - and( - eq(heartbeatRuns.companyId, companyId), - createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, - inArray(heartbeatRuns.status, [ - ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, - ]), - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function listProjectScopedWakeupIds( - companyId: string, - projectId: string, - database: Db = db, - ) { - const wakeIssueId = sql< - string | null - >`${agentWakeupRequests.payload} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await database - .selectDistinctOn([agentWakeupRequests.id], { - id: agentWakeupRequests.id, - }) - .from(agentWakeupRequests) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${wakeIssueId}`, - ), - ) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function cancelPendingWakeupsForBudgetScope( - scope: BudgetEnforcementScope, - ) { - return withCurrentBudgetEnforcement(db, scope, async (tx) => { - const now = new Date(); - let wakeupIds: string[] = []; - - if (scope.scopeType === "company") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else if (scope.scopeType === "agent") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - eq(agentWakeupRequests.agentId, scope.scopeId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else { - wakeupIds = await listProjectScopedWakeupIds( - scope.companyId, - scope.scopeId, - tx, - ); - } - - if (wakeupIds.length === 0) return 0; - - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: "Cancelled due to budget pause", - updatedAt: now, - }) - .where(and( - inArray(agentWakeupRequests.id, wakeupIds), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), - isNull(agentWakeupRequests.runId), - scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, - )); - - return wakeupIds.length; - }); - } - - type CancelRunOptions = { - budgetEnforcement?: BudgetEnforcementScope; - /** Optional board request identity, atomically reserved for native Stop. */ - cancellationRequestId?: string; - cancellationRequestedByUserId?: string | null; - errorCode?: string; - resultJson?: Record; - eventMessage?: string; - eventPayload?: Record; - /** Per-call graceful process shutdown window, bounded to a safe range. */ - terminationGraceMs?: number; - /** Caller is immediately scheduling an explicit successor path. */ - suppressImmediateRecovery?: boolean; - }; - - function cancellationTerminationGraceMs( - configuredGraceSec: number, - requestedGraceMs: number | undefined, - ) { - const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; - if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { - return configuredGraceMs; - } - return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); - } - - async function cancelRunInternal( - runId: string, - reason = "Cancelled by control plane", - options: CancelRunOptions = {}, - ) { - let run = await getRun(runId); - if (!run) throw notFound("Heartbeat run not found"); - if (options.cancellationRequestId) { - run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); - } - // The caller claim checked retry eligibility under both durable row locks. - // This is only a cancellation candidate: dispatch rechecks the coordinator - // under lock, and the final CAS requires its own unchanged acknowledged - // retry fence. Re-reading only retryable_failure here would strand replay. - const pendingNativeRetry = - run.runtimeMode === "native" && run.status === "failed" && ( - Boolean(options.cancellationRequestId) || await db - .select({ runId: nativeRunFinalizations.runId }) - .from(nativeRunFinalizations) - .where( - and( - eq(nativeRunFinalizations.runId, run.id), - eq(nativeRunFinalizations.companyId, run.companyId), - eq(nativeRunFinalizations.phase, "retryable_failure"), - ), - ) - .then((rows) => rows.length > 0) - ); - if ( - !pendingNativeRetry && - !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( - run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], - ) - ) - return run; - const agent = await getAgent(run.agentId); - const errorCode = options.errorCode ?? "cancelled"; - const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); - - const pendingProcessCancellation = processRunCancellationSettlements.get( - run.id, - ); - if (pendingProcessCancellation) { - await pendingProcessCancellation.settled; - if (pendingProcessCancellation.failed) - throw pendingProcessCancellation.error; - return getRun(run.id); - } - const running = runningProcesses.get(run.id); - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - const control = stopOwnership?.control; - // Capture the existing adapter owner before waiting on the run lock. Then - // atomically fence preparation and refresh the selected runtime, so Stop - // cannot miss a native handoff that won after its first read. - // Established legacy processes must still be stopped if the database is - // unavailable. Only native or not-yet-dispatched preparation needs this - // additional durable fence before its existing cancellation path. - if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { - const fence = async (tx: Db) => { - const [fenced] = await tx.update(heartbeatRuns).set({ - // Record handoff intent before native cancellation can finalize and release - // the run. Only its audited stop acknowledgement suppresses recovery. - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || - ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true - ? { reassignmentStopRequested: true } : {})}::jsonb || - ${JSON.stringify({ cancellation })}::jsonb || - jsonb_build_object('startupCancellation', jsonb_build_object( - 'requestedAt', ${new Date().toISOString()}::text, - 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.runtimeModeResolvedAt} is null - and ${heartbeatRuns.executionStage} = 'preparing' - and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) - ))`, - }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, - pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - ))).returning(); - return fenced ?? null; - }; - let fenced: typeof run | null; - try { - fenced = options.budgetEnforcement - ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) - : await fence(db); - } catch (error) { - stopOwnership?.release(); - throw error; - } - if (!fenced) { - stopOwnership?.release(); - return getRun(runId); - } - run = fenced; - } - const resultJson = { ...(agent - ? { - ...mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: parseObject(run.resultJson), - errorCode, - errorMessage: reason, - }), - ...(options.resultJson ?? {}), - } - : options.resultJson), cancellation }; - - try { - let releaseProcessCancellation: (() => void) | undefined; - const processCancellationSettlement = - run.runtimeMode !== "native" && - !control && - running - ? { - settled: new Promise((resolve) => { - releaseProcessCancellation = resolve; - }), - failed: false, - error: undefined as unknown, - } - : undefined; - if (processCancellationSettlement) { - // No await between joining an existing owner above and registering ours. - processRunCancellationSettlements.set( - run.id, - processCancellationSettlement, - ); - } - const cancellation = await (async () => { - try { - if (control) { - await db - .update(heartbeatRuns) - .set({ - error: reason, - errorCode, - // Stop may have read the row before adapter settlement saved - // a required-copyback receipt. Merge into current database JSON - // and leave those server-owned fields to the restore recorder. - resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ - ...resultJson, - ...(!running ? { executionCancellation: { - state: "requested", requestedAt: new Date().toISOString(), - } } : {}), - }, true), - updatedAt: new Date(), - }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ); - control.controller.abort(new Error(reason)); - } - let terminationSettled = false; - try { - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), - }); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM - // can leave commands in their separate process groups alive. - signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, - graceMs: cancellationTerminationGraceMs( - running.graceSec, - options.terminationGraceMs, - ), - }); - } - terminationSettled = true; - } finally { - if ( - (!processCancellationSettlement || terminationSettled) && - runningProcesses.get(run.id) === running - ) { - runningProcesses.delete(run.id); - } - } - - if (control) { - await waitForAdapterStop(control.settled, undefined, { - runId: run.id, - adapterType: agent?.adapterType, - runtimeMode: run.runtimeMode, - abortRequested: control.controller.signal.aborted, - }, control); - const stopped = await getRun(run.id); - if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { - if ( - parseObject(stopped.resultJson?.executionCancellation).state !== - "acknowledged" - ) { - throw conflict( - "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", - ); - } - // The owned adapter already finalized this run and its lifecycle. - // Do not replay the process cancellation side effects below. - return { run: stopped, updated: false }; - } - } - - const finishedAt = new Date(); - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : {}; - return await setRunStatusFromLive( - run.id, - "cancelled", - pendingNativeRetry - ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] - : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - { - finishedAt, - error: reason, - errorCode, - ...(resultJson || - Object.keys(persistedCancellationResult).length > 0 - ? { - resultJson: { - ...persistedCancellationResult, - ...(resultJson ?? {}), - // A scheduler placeholder has no process to acknowledge. - // Preserve its normal release policy instead of treating - // it as an operator stop of provider work. - ...(processCancellationSettlement && agent && running && ( - (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || - (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) - ) - ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: { - ...resultJson, - executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, - }, - errorCode, errorMessage: reason, - }) - : {}), - // The native cancellation helper may have advanced a durable - // pending intent to its acknowledged state after `run` was - // first read. Never let that stale snapshot overwrite the - // authoritative post-dispatch acknowledgement. - ...(Object.hasOwn( - persistedCancellationResult, - "nativeCancellation", - ) - ? { - nativeCancellation: - persistedCancellationResult.nativeCancellation, - } - : {}), - }, - } - : {}), - }, - undefined, - pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, - ); - } catch (error) { - if (processCancellationSettlement) { - processCancellationSettlement.failed = true; - processCancellationSettlement.error = error; - if (activeRunExecutions.has(run.id)) { - failedProcessRunCancellations.set( - run.id, - processCancellationSettlement, - ); - } - } - throw error; - } finally { - if (processCancellationSettlement) { - if ( - processRunCancellationSettlements.get(run.id) === - processCancellationSettlement - ) { - processRunCancellationSettlements.delete(run.id); - } - // Always settle waiters, including termination and DB errors. A - // failed Stop remains an error to its caller, never a cancellation - // receipt; the signal-bearing executor may then record failure. - releaseProcessCancellation?.(); - } - } - })(); - const cancelled = cancellation.run; - - if (cancellation.updated && cancelled) { - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: cancelled.finishedAt ?? new Date(), - error: reason, - }); - await appendRunEvent(cancelled, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: options.eventMessage ?? "run cancelled", - payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, - }); - await releaseIssueExecutionAndPromote(cancelled, { - suppressImmediateRecovery: options.suppressImmediateRecovery, - }); - await finalizeAgentStatus(run.agentId, "cancelled", undefined, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }); - await startNextQueuedRunForAgent(run.agentId); - } - return cancelled; - } finally { - stopOwnership?.release(); - } - } - - async function cancelActiveForAgentInternal( - agentId: string, - reason = "Cancelled due to agent pause", - errorCode = "cancelled", - ) { - const agent = await getAgent(agentId); - const runs = await db - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - ), - ); - - for (const run of runs) { - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - try { - if (stopOwnership?.control) { - await cancelRunInternal(run.id, reason, { errorCode }); - continue; - } - if (run.runtimeMode === "native") { - await db.update(heartbeatRuns).set({ resultJson: - sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`, - }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - }); - } - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : parseObject(run.resultJson); - await setRunStatus(run.id, "cancelled", { - finishedAt: new Date(), - error: reason, - errorCode, - resultJson: { - ...persistedCancellationResult, - ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: persistedCancellationResult, errorCode, errorMessage: reason, - }) : {}), - cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason), - }, - }); - - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: new Date(), - error: reason, - }); - - const running = runningProcesses.get(run.id); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - graceMs: Math.max(1, running.graceSec) * 1000, - }); - } - runningProcesses.delete(run.id); - await releaseIssueExecutionAndPromote(run); - } finally { - stopOwnership?.release(); - } - } - - return runs.length; - } - - async function cancelPendingWakeupsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - if (uniqueAgentIds.length === 0) return 0; - - const now = new Date(); - const wakeupIds = await db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - inArray(agentWakeupRequests.agentId, uniqueAgentIds), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - - if (wakeupIds.length === 0) return 0; - - await db - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: reason, - updatedAt: now, - }) - .where(inArray(agentWakeupRequests.id, wakeupIds)); - - return wakeupIds.length; - } - - async function cancelInvocationsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - let runsCancelled = 0; - for (const agentId of uniqueAgentIds) { - runsCancelled += await cancelActiveForAgentInternal(agentId, reason); - } - const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( - uniqueAgentIds, - reason, - ); - return { - agentIds: uniqueAgentIds, - runsCancelled, - wakeupsCancelled, - }; - } - - async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { - const runIds = scope.scopeType === "project" - ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) - : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, scope.companyId), - scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, - scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - )).then((rows) => rows.map((row) => row.id)); - for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); - await cancelPendingWakeupsForBudgetScope(scope); - } - return { waitForRunExecutionDrain: async ( runId: string, diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts new file mode 100644 index 00000000000..7af48aeee59 --- /dev/null +++ b/server/src/services/heartbeat/run-control.test.ts @@ -0,0 +1,310 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import { createHeartbeatRunPreparation } from "./run-preparation.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; +import { createHeartbeatRunControl, type HeartbeatRunControlDependencies } from "./run-control.js"; + +vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); +vi.mock("../local-service-supervisor.js", async (importOriginal) => ({ + ...await importOriginal(), + terminateLocalService: vi.fn(async () => undefined), +})); + +type Run = typeof heartbeatRuns.$inferSelect; +function callbacks(db: Db) { + const state = createHeartbeatRunState(db); + const lifecycleDependencies = { + ...state, + ...createHeartbeatRunPreparation(db), + issuesSvc: { + addComment: vi.fn(), + listReviewAttention: vi.fn(), + }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, + companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, + treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), + getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), + emitTerminalAgentTaskRun: vi.fn(), + budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + const lifecycle = createHeartbeatLifecycle(db, lifecycleDependencies); + return { + ...state, + ...lifecycle, + getRun: vi.fn(state.getRun), + getAgent: vi.fn(state.getAgent), + setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive), + setRunStatus: vi.fn(lifecycle.setRunStatus), + setWakeupStatus: vi.fn(lifecycle.setWakeupStatus), + appendRunEvent: vi.fn(lifecycle.appendRunEvent), + finalizeAgentStatus: vi.fn(), + options: { closeWarmNativeSessionsForRun: vi.fn(async () => ({ closed: 1, busy: 0, failed: 0 })) }, + envOrchestrator: { releaseForRun: vi.fn(async () => ({ released: [], errors: [] })) }, + enqueueWakeup: lifecycleDependencies.enqueueWakeup, + startNextQueuedRunForAgent: vi.fn(), + sweepPendingCleanupLeases: vi.fn(), + timerClaimWasFirstHeartbeat: vi.fn(() => undefined), + wakeQueue: { releaseIssueExecution: vi.fn(async () => ({ outcome: { kind: "released" }, postCommitEffects: [] })) }, + applyWakeQueuePostCommitEffects: vi.fn(async () => undefined), + getSchedulingSuppression: vi.fn(async () => ({ suppressed: false })), + activeRunExecutions: new Set(), + processRunCancellationSettlements: new Map; failed: boolean; error?: unknown }>(), + failedProcessRunCancellations: new Map; failed: boolean; error?: unknown }>(), + } satisfies HeartbeatRunControlDependencies; +} +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected run-control database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +describe("heartbeat run-control module boundary", () => { + afterEach(() => adapterExecutionControls.clear()); + + it("constructs without database access, cancellation, or cleanup", () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + createHeartbeatRunControl(database.db, deps); + expect(database.access).not.toHaveBeenCalled(); + for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + }); + + it("keeps an already completed run out of cancellation effects", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "finished", status: "succeeded", runtimeMode: "legacy" } as Run; + deps.getRun.mockResolvedValue(run); + expect(await createHeartbeatRunControl(database.db, deps).cancelRunInternal(run.id)).toBe(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("keeps a suppressed saved-input scan out of the database and wake admission", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true }); + await createHeartbeatRunControl(database.db, deps).resumeExecutionWaitComments(); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("ignores an empty cancellation batch", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + expect(await createHeartbeatRunControl(database.db, deps).cancelInvocationsForAgentsInternal(["", ""], "pause")).toEqual({ agentIds: [], runsCancelled: 0, wakeupsCancelled: 0 }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + }); + + it("waits for an owned adapter before admitting saved comments", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "owned", status: "cancelled", runtimeMode: "legacy" } as Run; + adapterExecutionControls.set(run.id, {} as never); + await createHeartbeatRunControl(database.db, deps).resumeRemoteStopComments(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("retains leases while native execution ownership is unverified", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "held", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + }); + + it("uses the persisted terminal outcome when choosing warm resource retention", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "failed", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" }); + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ heartbeatRunId: "failed", status: "failed", providerResourceDisposition: "stop_and_retain" })); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("defers destruction when a warm session remains busy", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 }); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" }); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(database.access).not.toHaveBeenCalled(); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping run-control database tests: ${support.reason}`); +describe.skipIf(!support.supported)("heartbeat run-control database wiring", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-run-control-module"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + beforeEach(async () => { + vi.mocked(terminateLocalService).mockReset().mockResolvedValue(undefined); + await db.execute(sql`truncate table companies restart identity cascade`); + }); + afterEach(() => runningProcesses.clear()); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Run control", issuePrefix: "RC" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Controlled agent", adapterType: "codex_local" }).returning(); + return { company, agent }; + } + async function liveRun(companyId: string, agentId: string) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "legacy", runtimeModeResolvedAt: new Date() }).returning(); + runningProcesses.set(run.id, { child: { pid: 424242 } as never, graceSec: 4, processGroupId: 424242 }); + return run; + } + + it("reports a missing run without entering cancellation", async () => { + const deps = callbacks(db); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found"); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("joins concurrent Stop requests across factories through the shared process barrier", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + let releaseTermination!: () => void; + const terminating = new Promise(resolve => { releaseTermination = resolve; }); + vi.mocked(terminateLocalService).mockReturnValue(terminating); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop", { terminationGraceMs: 1 }); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop again"); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + releaseTermination(); + const results = await Promise.all([first, second]); + expect(results.map(result => result?.status)).toEqual(["cancelled", "cancelled"]); + expect(deps.setRunStatusFromLive).toHaveBeenCalledTimes(1); + expect(deps.appendRunEvent).toHaveBeenCalledTimes(1); + expect(deps.finalizeAgentStatus).toHaveBeenCalledTimes(1); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(false); + expect(terminateLocalService).toHaveBeenCalledWith({ pid: 424242, processGroupId: 424242 }, { forceAfterMs: 100, signal: "SIGINT" }); + }); + + it("settles every joined Stop caller with the same termination failure", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + deps.activeRunExecutions.add(run.id); + let rejectTermination!: (error: Error) => void; + vi.mocked(terminateLocalService).mockReturnValue(new Promise((_resolve, reject) => { rejectTermination = reject; })); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + const results = Promise.allSettled([first, second]); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + const failure = new Error("provider did not stop"); + rejectTermination(failure); + expect(await results).toEqual([{ status: "rejected", reason: failure }, { status: "rejected", reason: failure }]); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect((await deps.getRun(run.id))?.status).toBe("running"); + }); + + it("retains failed Stop evidence and the owned process until a later request succeeds", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + deps.activeRunExecutions.add(run.id); + const failure = new Error("termination failed"); + vi.mocked(terminateLocalService).mockRejectedValueOnce(failure); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).rejects.toBe(failure); + expect(deps.failedProcessRunCancellations.get(run.id)).toMatchObject({ failed: true, error: failure }); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(true); + expect((await deps.getRun(run.id))?.status).toBe("running"); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" }); + expect(terminateLocalService).toHaveBeenCalledTimes(2); + // The executor still owns removal of this evidence after it observes Stop. + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + }); + + it("deduplicates agent batches and cancels only unclaimed pending wakes", async () => { + const { company, agent } = await fixture(); + const [otherAgent] = await db.insert(agents).values({ companyId: company.id, name: "Other agent" }).returning(); + const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "succeeded" }).returning(); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "deferred_issue_execution" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", runId: finished.id }, + { companyId: company.id, agentId: otherAgent.id, source: "assignment", status: "queued" }, + ]).returning(); + const deps = callbacks(db); + expect(await createHeartbeatRunControl(db, deps).cancelInvocationsForAgentsInternal([agent.id, "", agent.id], "pause")).toEqual({ agentIds: [agent.id], runsCancelled: 0, wakeupsCancelled: 2 }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "cancelled", "claimed", "queued", "queued"]); + expect(deps.getAgent).toHaveBeenCalledTimes(1); + }); + + it("rechecks stale budget enforcement before cancelling runs or queued wakes", async () => { + const { company, agent } = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", runtimeMode: "legacy" }).returning(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }).returning(); + const deps = callbacks(db); + await createHeartbeatRunControl(db, deps).cancelBudgetScopeWork({ companyId: company.id, scopeType: "agent", scopeId: agent.id, enforcement: { policyId: randomUUID(), version: 1 } }); + expect((await deps.getRun(run.id))?.status).toBe("queued"); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued"); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(terminateLocalService).not.toHaveBeenCalled(); + }); + + it("limits project budget cancellation by company, effective project, and admission cutoff", async () => { + const { company, agent } = await fixture(); + const [otherCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); + const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Other company agent" }).returning(); + const projectRows = await db.insert(projects).values([ + { companyId: company.id, name: "Target" }, { companyId: company.id, name: "Override" }, + ]).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: projectRows[0].id, title: "Bound task", issueNumber: 1, identifier: "RC-1" }).returning(); + const old = new Date("2026-01-01T00:00:00Z"); + const cutoff = new Date("2026-01-02T00:00:00Z"); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id, projectId: projectRows[1].id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: cutoff, payload: { projectId: projectRows[0].id } }, + { companyId: otherCompany.id, agentId: otherAgent.id, source: "assignment", status: "queued", createdAt: old, payload: { projectId: projectRows[0].id } }, + ]).returning(); + await createHeartbeatRunControl(db, callbacks(db)).cancelBudgetScopeWork({ companyId: company.id, scopeType: "project", scopeId: projectRows[0].id, createdBefore: cutoff }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "queued", "queued", "queued"]); + }); +}); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts new file mode 100644 index 00000000000..dd3f9d8dc3c --- /dev/null +++ b/server/src/services/heartbeat/run-control.ts @@ -0,0 +1,1360 @@ +import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; +import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js"; +import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js"; +import { + type WakeupOptions, +} from "./run-preparation.js"; +import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; +import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; +import { + nativeRetryCancellationCommitCondition, + claimCancellationRequest, +} from "../native-runtime/native-cancellation-request.js"; +import { + canContinueCancelledRun, + readRunCancellation, + requestedRunCancellation, +} from "../run-cancellation.js"; +import { settleSlackConversation } from "../slack-conversation-lifecycle.js"; +import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; +import { + isAcknowledgedNativeStop, + acknowledgedNativeStopExecutionHasStopped, +} from "../acknowledged-native-stop.js"; +import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js"; +import { + hasRemoteTerminationReceipt, + remoteExecutionHasStopped, + stoppedRemoteCleanupScopes, +} from "../remote-execution-termination.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "../explicit-native-continuation.js"; +import { isCancelledNativeStartup } from "../cancelled-native-startup.js"; +import { + executionBlockerPredicate, + getExecutionBlocker, +} from "../execution-blocker.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + runUsedConversationAdapter, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { + adapterExecutionControls, + captureAdapterStopOwnership, + waitForAdapterStop, +} from "../adapter-execution-control.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + exists, + gt, + inArray, + isNull, + isNotNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + type EnvironmentLeaseStatus, +} from "@paperclipai/shared"; +import { + agentWakeupRequests, + companies, + environmentLeases, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { + conflict, + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, + withQueuedCommentIdsInRunContext, +} from "../issue-queued-comment-queue.js"; +import { + cancelNativeSession, + closeWarmNativeSessionsForEnvironment, +} from "../native-runtime/index.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { parseObject } from "../../adapters/utils.js"; +import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js"; +import { + withCurrentBudgetEnforcement, + type BudgetEnforcementScope, +} from "../budgets.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { deriveCommentId } from "../../modules/run-dispatch/index.js"; +import { WakeQueueApplicationError } from "../../modules/wake-queue/index.js"; +import { isUuidLike } from "@paperclipai/shared"; +import { + type ProviderResourceDisposition, +} from "../environment-runtime.js"; + +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; +import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js"; +import type { createHeartbeatQueue } from "./queue.js"; +import type { createHeartbeatRecovery } from "./recovery.js"; +import type { createWakeQueue, PostCommitEffect as WakeQueuePostCommitEffect } from "../../modules/wake-queue/index.js"; +import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; + +type ProcessRunCancellationSettlement = { + settled: Promise; + failed: boolean; + error?: unknown; +}; + +/** Cancellation shares executor ownership and delegates lifecycle and admission effects. */ +export interface HeartbeatRunControlDependencies extends Pick, + "getRun" | "getAgent">, + Pick, + "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">, + Pick { + options: { + closeWarmNativeSessionsForRun?: (input: { + runId: string; + reason: string; + }) => Promise<{ closed: number; busy: number; failed: number }>; + }; + envOrchestrator: Pick, "releaseForRun">; + enqueueWakeup: ReturnType["enqueueWakeup"]; + startNextQueuedRunForAgent: ReturnType["startNextQueuedRunForAgent"]; + sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; + timerClaimWasFirstHeartbeat: ReturnType["timerClaimWasFirstHeartbeat"]; + wakeQueue: Pick, "releaseIssueExecution">; + applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; + getSchedulingSuppression: () => Promise<{ suppressed: boolean }>; + // Scheduler and route service instances must retain the same in-process barriers. + activeRunExecutions: Set; + processRunCancellationSettlements: Map; + failedProcessRunCancellations: Map; +} + +export function leaseReleaseStatusForRunStatus( + status: string | null | undefined, +): Extract { + if (status === "cancelled") return "expired"; + return status === "failed" || status === "timed_out" ? "failed" : "released"; +} + +export function providerResourceDispositionForTerminalRun( + desired: ProviderResourceDisposition | undefined, + status: string | null | undefined, +): ProviderResourceDisposition | undefined { + if (desired !== "keep_running") return desired; + return status === "succeeded" ? desired : "stop_and_retain"; +} + +export async function terminateHeartbeatRunProcess(input: { + pid: number | null | undefined; + processGroupId: number | null | undefined; + graceMs?: number; + signal?: NodeJS.Signals; +}) { + const pid = input.pid ?? null; + const processGroupId = input.processGroupId ?? null; + if (typeof pid !== "number" && typeof processGroupId !== "number") return; + + await terminateLocalService( + { + pid: + typeof pid === "number" && Number.isInteger(pid) && pid > 0 + ? pid + : (processGroupId ?? 0), + processGroupId: + typeof processGroupId === "number" && + Number.isInteger(processGroupId) && + processGroupId > 0 + ? processGroupId + : null, + }, + { forceAfterMs: input.graceMs, signal: input.signal }, + ); +} + +export async function cancelHeartbeatNativeRun(input: { + db: Db; + runId: string; + reason: string; + runtimeMode: string | null; + cancellationRequestId?: string; + cancel?: ( + runId: string, + reason: string, + options: { db: Db; scope: "run"; cancellationRequestId?: string }, + ) => Promise<{ decision: unknown | null; auditId: string | null }>; +}) { + if (input.runtimeMode !== "native") { + return { decision: null, auditId: null }; + } + const cancellation = input.cancel + ? await input.cancel(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }) + : await cancelNativeSession(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }); + if (!cancellation.decision || !cancellation.auditId) { + throw new Error("native_cancellation_outcome_not_audited"); + } + return cancellation; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export type CancelRunOptions = { + budgetEnforcement?: BudgetEnforcementScope; + /** Optional board request identity, atomically reserved for native Stop. */ + cancellationRequestId?: string; + cancellationRequestedByUserId?: string | null; + errorCode?: string; + resultJson?: Record; + eventMessage?: string; + eventPayload?: Record; + /** Per-call graceful process shutdown window, bounded to a safe range. */ + terminationGraceMs?: number; + /** Caller is immediately scheduling an explicit successor path. */ + suppressImmediateRecovery?: boolean; + }; + +/** Run control binds cancellation, resource release, and saved-input resumption. */ +export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunControlDependencies) { + const { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + } = dependencies; + + async function releaseEnvironmentLeasesForRun(input: { + runId: string; + companyId: string; + agentId: string; + status: string | null | undefined; + failureReason?: string | null; + providerResourceDisposition?: ProviderResourceDisposition; + nativeLifecycleTelemetry?: { + provider: string; + harness: string; + lifecycleMode: "per_turn" | "warm"; + sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; + }; + }) { + const leaseOwnerRun = await getRun(input.runId); + if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; + // Recovery can finish workspace copy-back outside the executor's finally. + // Successful copy-back does not earn warm retention for a failed turn. + const status = leaseOwnerRun?.status ?? input.status; + const providerResourceDisposition = providerResourceDispositionForTerminalRun( + input.providerResourceDisposition, + status, + ); + if (providerResourceDisposition === "destroy") { + const closeResult = await ( + options.closeWarmNativeSessionsForRun ?? + (async ({ runId, reason }) => { + const environmentIds = await db + .selectDistinct({ environmentId: environmentLeases.environmentId }) + .from(environmentLeases) + .where( + and( + eq(environmentLeases.heartbeatRunId, runId), + eq(environmentLeases.status, "active"), + ), + ) + .then((rows) => + rows.flatMap((row) => + typeof row.environmentId === "string" && + row.environmentId.length > 0 + ? [row.environmentId] + : [], + ), + ); + const aggregate = { closed: 0, busy: 0, failed: 0 }; + for (const environmentId of environmentIds) { + const result = await closeWarmNativeSessionsForEnvironment({ + environmentId, + reason, + }); + aggregate.closed += result.closed; + aggregate.busy += result.busy; + aggregate.failed += result.failed; + } + return aggregate; + }) + )({ + runId: input.runId, + reason: "terminal heartbeat run destroyed its environment lease", + }).catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to close warm native sessions before environment lease destruction", + ); + return { closed: 0, busy: 0, failed: 1 }; + }); + // A failed remote checkpoint cannot veto destruction of the isolated + // sandbox after the run stopped. Provider destruction supplies the exit + // proof; it does not turn the interrupted checkpoint into a success. + const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && + ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) + ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), + )) + : []; + const canDestroyRemote = remoteLeases.length > 0 && + remoteLeases.every(lease => lease.provider && lease.provider !== "local"); + if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { + logger.warn( + { runId: input.runId, warmNativeSessions: closeResult }, + "deferred environment lease destruction until warm native sessions close", + ); + return; + } + } + const releaseResult = await envOrchestrator + .releaseForRun({ + heartbeatRunId: input.runId, + companyId: input.companyId, + agentId: input.agentId, + status: leaseReleaseStatusForRunStatus(status), + failureReason: input.failureReason ?? undefined, + providerResourceDisposition, + nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, + }) + .catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to release environment leases for heartbeat run", + ); + return null; + }); + for (const releaseError of releaseResult?.errors ?? []) { + logger.warn( + { + err: releaseError.error, + leaseId: releaseError.leaseId, + runId: input.runId, + }, + "failed to release environment lease for heartbeat run", + ); + } + await acknowledgeRemoteStop(input.runId, input.companyId); + } + + async function acknowledgeRemoteStop(runId: string, companyId: string) { + // The provider receipt arrives after adapter settlement. A remote ACP child + // has no host PID, so only this target-aware boundary can acknowledge Stop. + const stopped = await getRun(runId); + if (stopped?.runtimeMode === "native") { + const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); + for (const remoteCleanupScope of scopes ?? []) { + completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); + } + } + if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && + parseObject(stopped.resultJson?.executionCancellation).state === "requested" && + await runUsedConversationAdapter(db, stopped) && + await remoteExecutionHasStopped(db, companyId, runId)) { + await db.update(heartbeatRuns).set({ + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ + executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), + state: "acknowledged", acknowledgedAt: new Date().toISOString(), + proof: "provider_termination_receipt" }, + conversationContinuation: CONVERSATION_CONTINUATION_POLICY, + })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"))); + } + } + + async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { + if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; + const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt + ? await db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + const cancelledPreparation = run.runtimeMode === "legacy" && + await isCancelledNativeStartup(db, run, preparationCoordinator); + if (run.runtimeMode !== "native" && !cancelledPreparation && + parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && + !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; + const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); + if (!issueId) return; + const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; + const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, + leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + // Stop ends one response. Saved user input can enter ordinary admission + // once its executor settles; it does not need a manufactured crash incident. + // The ordinary path still enforces task holds, ownership, and native session + // cleanup before starting a provider. + const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && + !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && + ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && + await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + const legacyContinuation = run.runtimeMode === "legacy" && + hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; + const pending = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + requestId ? eq(agentWakeupRequests.id, requestId) : undefined, + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); + for (const wake of pending) { + if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; + let payload = parseObject(wake.payload); + if (payload.queuedCommentInterrupt) { + await resumeQueuedCommentInterrupt(wake.companyId, wake.id); + continue; + } + let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + let commentId = deriveCommentId(context, payload); + let requestedByActorId = wake.requestedByActorId; + const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; + if (run.runtimeMode === "native" || cancelledPreparation) { + const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, + queuedCommentIdsFromWakePayload(payload)); + if (!ids.length) continue; + payload = withQueuedCommentIdsInWakePayload(payload, ids); + context = withQueuedCommentIdsInRunContext(context, ids); + commentId = ids.at(-1)!; + // A coalesced queue can contain several authors. Its saved comments, + // not the outer wake's first author, authorize the remaining input. + const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), + )); + requestedByActorId = author?.id ?? null; + } + if (legacyContinuation || stoppedNativeContinuation) { + if (!commentId || !run.finishedAt || !requestedByActorId || + !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), + sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), + eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), + isNull(issueComments.createdByRunId), + stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), + )); + if (!comment?.body.trim()) continue; + } else { + let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; + const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, + agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, + reason, commentId, successorRunId: randomUUID(), dryRun: true, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + onBlocked: (reason, message) => { wait = { reason, message }; }, + }); + if (!admitted) { + await db.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + continue; + } + } + // Re-enter ordinary admission with the original user's authority. It + // atomically adopts the deferred comments and still applies every gate. + await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], + reason, payload, contextSnapshot: context, + requestedByActorType: "user", requestedByActorId, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); + break; + } + } + + async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { + return resumeSavedLegacyComments(companyId, queueId, true, opts); + } + + async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { + const [wake] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!wake) return; + const payload = parseObject(wake.payload); + let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); + let commentIds = queuedCommentIdsFromWakePayload(payload); + const issueId = readNonEmptyString(payload.issueId); + if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; + const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); + if (!commentIds.length && !response) return; + // Resolved cards are immutable input. Only an explicit Interrupt click can + // authorize continuation across a stopped execution; ordinary completion + // uses normal deferred-wake promotion. + if (response && !interrupted) return; + if (!interrupted) { + commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); + if (!commentIds.length) return; + // The queue itself may have begun as a system wake. The saved human + // comment, not that wake's origin or mutable caller payload, is authority. + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), + isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), + )).orderBy(desc(issueComments.createdAt)).limit(1); + if (!comment?.body.trim() || !comment.authorUserId) return; + actorId = comment.authorUserId; + } + if (!actorId) return; + const agent = await getAgent(wake.agentId); + if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (active) return; + if (interrupted && opts?.retryCleanup) { + // Only the HTTP click grants an extra cleanup attempt. Periodic retries + // reuse the intent to deliver, never a fresh provider teardown budget. + const sourceRun = await db.transaction(async tx => { + const [task] = await tx.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.id, issueId), + )).for("update"); + const [current] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, + )); + if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || + !current || (!queuedCommentIdsFromWakePayload(current.payload).length && + !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; + const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (successor) return null; + const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); + const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), + eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )).then(rows => rows[0]) : null; + if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; + // Older ephemeral leases recorded successful cleanup without a provider + // receipt. Re-verify them through the recorded teardown path; a timestamp + // alone never certifies termination. Retained/reusable resources stay put. + const historical = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), + inArray(environmentLeases.status, ["released", "expired", "failed"]), + )).for("update"); + for (const lease of historical) { + if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; + // Provider resource IDs identify physical sandboxes. A lease in any + // company can still own this resource; never destroy it on behalf of + // this company. This existence-only guard exposes no foreign data. + const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), + )).limit(1); + if (otherOwner) continue; + await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) + .where(eq(environmentLeases.id, lease.id)); + } + return run; + }); + if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { + companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", + } }); + } + const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); + delete deliveryPayload.queuedCommentInterrupt; + await enqueueWakeup(wake.agentId, { + source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: deliveryPayload, contextSnapshot: response + ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, + responsibleUserId: actorId } + : withQueuedCommentIdsInRunContext({ + issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, + }, commentIds), + requestedByActorType: "user", requestedByActorId: actorId, + ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), + issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, + idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, + }, queueId); + } + + async function resumeExecutionWaitComments() { + if ((await getSchedulingSuppression()).suppressed) return; + const waits = await db.select({ wake: agentWakeupRequests }) + .from(agentWakeupRequests) + .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), + sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, + eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) + .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) + .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), + executionBlockerPredicate(), + ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`${agentWakeupRequests.payload}->'executionWait' is not null`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + notInArray(issues.status, ["done", "cancelled"]))) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + const seen = new Set(); + for (const { wake } of waits) { + const issueId = String(wake.payload?.issueId); + if (seen.has(issueId)) continue; + seen.add(issueId); + // Advance the cursor even for invalid evidence so one damaged task cannot + // starve later requests in this bounded scan. Preserve concurrent edits. + const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + )).returning({ id: agentWakeupRequests.id }); + if (!claimed) continue; + // Match normal admission's deterministic current blocker selection. An + // arbitrary historical action must not choose the retry's source run. + const blocker = await getExecutionBlocker(db, wake.companyId, issueId); + const sourceId = blocker?.runId; + if (!sourceId || !isUuidLike(sourceId)) continue; + const run = await getRun(sourceId, { includeExecutionEvidence: true }); + if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; + if (canContinueCancelledRun(run)) { + await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); + }); + continue; + } + await resumeRemoteStopComments(run, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); + }); + } + } + + async function releaseIssueExecutionAndPromote( + run: Pick, + options: { suppressImmediateRecovery?: boolean } = {}, + ) { + try { + const source = await getRun(run.id); + const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ + companyId: run.companyId, + runId: run.id, + now: new Date(), + // A durable authentication card owns recovery. This covers the review path, + // while continuation classification blocks periodic generic retries. + suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), + }); + await applyWakeQueuePostCommitEffects(postCommitEffects); + const completed = await getRun(run.id); + const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) + ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; + if (completed?.status === "succeeded" && issueId) { + await settleSlackConversation(db, run.companyId, issueId).catch((err) => { + logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); + }); + } + } catch (error) { + if ( + error instanceof WakeQueueApplicationError && + error.code === "responsible_user_unresolved" + ) { + // Every other `responsible_user_unresolved` HttpError in this file + // carries `code` inside its own `details`; match that shape here too. + throw new HttpError(422, error.message, { code: error.code, ...error.details }); + } + throw error; + } + } + + async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { + const runIssueId = sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await db + .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) + .from(heartbeatRuns) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${runIssueId}`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function listProjectScopedWakeupIds( + companyId: string, + projectId: string, + database: Db = db, + ) { + const wakeIssueId = sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await database + .selectDistinctOn([agentWakeupRequests.id], { + id: agentWakeupRequests.id, + }) + .from(agentWakeupRequests) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${wakeIssueId}`, + ), + ) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function cancelPendingWakeupsForBudgetScope( + scope: BudgetEnforcementScope, + ) { + return withCurrentBudgetEnforcement(db, scope, async (tx) => { + const now = new Date(); + let wakeupIds: string[] = []; + + if (scope.scopeType === "company") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else if (scope.scopeType === "agent") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + eq(agentWakeupRequests.agentId, scope.scopeId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else { + wakeupIds = await listProjectScopedWakeupIds( + scope.companyId, + scope.scopeId, + tx, + ); + } + + if (wakeupIds.length === 0) return 0; + + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: "Cancelled due to budget pause", + updatedAt: now, + }) + .where(and( + inArray(agentWakeupRequests.id, wakeupIds), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + isNull(agentWakeupRequests.runId), + scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, + )); + + return wakeupIds.length; + }); + } + + function cancellationTerminationGraceMs( + configuredGraceSec: number, + requestedGraceMs: number | undefined, + ) { + const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; + if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { + return configuredGraceMs; + } + return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); + } + + async function cancelRunInternal( + runId: string, + reason = "Cancelled by control plane", + options: CancelRunOptions = {}, + ) { + let run = await getRun(runId); + if (!run) throw notFound("Heartbeat run not found"); + if (options.cancellationRequestId) { + run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); + } + // The caller claim checked retry eligibility under both durable row locks. + // This is only a cancellation candidate: dispatch rechecks the coordinator + // under lock, and the final CAS requires its own unchanged acknowledged + // retry fence. Re-reading only retryable_failure here would strand replay. + const pendingNativeRetry = + run.runtimeMode === "native" && run.status === "failed" && ( + Boolean(options.cancellationRequestId) || await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.companyId, run.companyId), + eq(nativeRunFinalizations.phase, "retryable_failure"), + ), + ) + .then((rows) => rows.length > 0) + ); + if ( + !pendingNativeRetry && + !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( + run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], + ) + ) + return run; + const agent = await getAgent(run.agentId); + const errorCode = options.errorCode ?? "cancelled"; + const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); + + const pendingProcessCancellation = processRunCancellationSettlements.get( + run.id, + ); + if (pendingProcessCancellation) { + await pendingProcessCancellation.settled; + if (pendingProcessCancellation.failed) + throw pendingProcessCancellation.error; + return getRun(run.id); + } + const running = runningProcesses.get(run.id); + const stopOwnership = + run.runtimeMode !== "native" + ? captureAdapterStopOwnership(run.id) + : undefined; + const control = stopOwnership?.control; + // Capture the existing adapter owner before waiting on the run lock. Then + // atomically fence preparation and refresh the selected runtime, so Stop + // cannot miss a native handoff that won after its first read. + // Established legacy processes must still be stopped if the database is + // unavailable. Only native or not-yet-dispatched preparation needs this + // additional durable fence before its existing cancellation path. + if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { + const fence = async (tx: Db) => { + const [fenced] = await tx.update(heartbeatRuns).set({ + // Record handoff intent before native cancellation can finalize and release + // the run. Only its audited stop acknowledgement suppresses recovery. + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || + ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true + ? { reassignmentStopRequested: true } : {})}::jsonb || + ${JSON.stringify({ cancellation })}::jsonb || + jsonb_build_object('startupCancellation', jsonb_build_object( + 'requestedAt', ${new Date().toISOString()}::text, + 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' + and ${heartbeatRuns.runtimeModeResolvedAt} is null + and ${heartbeatRuns.executionStage} = 'preparing' + and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) + ))`, + }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, + pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + ))).returning(); + return fenced ?? null; + }; + let fenced: typeof run | null; + try { + fenced = options.budgetEnforcement + ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) + : await fence(db); + } catch (error) { + stopOwnership?.release(); + throw error; + } + if (!fenced) { + stopOwnership?.release(); + return getRun(runId); + } + run = fenced; + } + const resultJson = { ...(agent + ? { + ...mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: parseObject(run.resultJson), + errorCode, + errorMessage: reason, + }), + ...(options.resultJson ?? {}), + } + : options.resultJson), cancellation }; + + try { + let releaseProcessCancellation: (() => void) | undefined; + const processCancellationSettlement = + run.runtimeMode !== "native" && + !control && + running + ? { + settled: new Promise((resolve) => { + releaseProcessCancellation = resolve; + }), + failed: false, + error: undefined as unknown, + } + : undefined; + if (processCancellationSettlement) { + // No await between joining an existing owner above and registering ours. + processRunCancellationSettlements.set( + run.id, + processCancellationSettlement, + ); + } + const cancellation = await (async () => { + try { + if (control) { + await db + .update(heartbeatRuns) + .set({ + error: reason, + errorCode, + // Stop may have read the row before adapter settlement saved + // a required-copyback receipt. Merge into current database JSON + // and leave those server-owned fields to the restore recorder. + resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ + ...resultJson, + ...(!running ? { executionCancellation: { + state: "requested", requestedAt: new Date().toISOString(), + } } : {}), + }, true), + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ); + control.controller.abort(new Error(reason)); + } + let terminationSettled = false; + try { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), + }); + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid, + processGroupId: running.processGroupId, + // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM + // can leave commands in their separate process groups alive. + signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, + graceMs: cancellationTerminationGraceMs( + running.graceSec, + options.terminationGraceMs, + ), + }); + } + terminationSettled = true; + } finally { + if ( + (!processCancellationSettlement || terminationSettled) && + runningProcesses.get(run.id) === running + ) { + runningProcesses.delete(run.id); + } + } + + if (control) { + await waitForAdapterStop(control.settled, undefined, { + runId: run.id, + adapterType: agent?.adapterType, + runtimeMode: run.runtimeMode, + abortRequested: control.controller.signal.aborted, + }, control); + const stopped = await getRun(run.id); + if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { + if ( + parseObject(stopped.resultJson?.executionCancellation).state !== + "acknowledged" + ) { + throw conflict( + "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", + ); + } + // The owned adapter already finalized this run and its lifecycle. + // Do not replay the process cancellation side effects below. + return { run: stopped, updated: false }; + } + } + + const finishedAt = new Date(); + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : {}; + return await setRunStatusFromLive( + run.id, + "cancelled", + pendingNativeRetry + ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] + : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + { + finishedAt, + error: reason, + errorCode, + ...(resultJson || + Object.keys(persistedCancellationResult).length > 0 + ? { + resultJson: { + ...persistedCancellationResult, + ...(resultJson ?? {}), + // A scheduler placeholder has no process to acknowledge. + // Preserve its normal release policy instead of treating + // it as an operator stop of provider work. + ...(processCancellationSettlement && agent && running && ( + (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || + (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) + ) + ? mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: { + ...resultJson, + executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, + }, + errorCode, errorMessage: reason, + }) + : {}), + // The native cancellation helper may have advanced a durable + // pending intent to its acknowledged state after `run` was + // first read. Never let that stale snapshot overwrite the + // authoritative post-dispatch acknowledgement. + ...(Object.hasOwn( + persistedCancellationResult, + "nativeCancellation", + ) + ? { + nativeCancellation: + persistedCancellationResult.nativeCancellation, + } + : {}), + }, + } + : {}), + }, + undefined, + pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, + ); + } catch (error) { + if (processCancellationSettlement) { + processCancellationSettlement.failed = true; + processCancellationSettlement.error = error; + if (activeRunExecutions.has(run.id)) { + failedProcessRunCancellations.set( + run.id, + processCancellationSettlement, + ); + } + } + throw error; + } finally { + if (processCancellationSettlement) { + if ( + processRunCancellationSettlements.get(run.id) === + processCancellationSettlement + ) { + processRunCancellationSettlements.delete(run.id); + } + // Always settle waiters, including termination and DB errors. A + // failed Stop remains an error to its caller, never a cancellation + // receipt; the signal-bearing executor may then record failure. + releaseProcessCancellation?.(); + } + } + })(); + const cancelled = cancellation.run; + + if (cancellation.updated && cancelled) { + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: cancelled.finishedAt ?? new Date(), + error: reason, + }); + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: options.eventMessage ?? "run cancelled", + payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, + }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: options.suppressImmediateRecovery, + }); + await finalizeAgentStatus(run.agentId, "cancelled", undefined, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }); + await startNextQueuedRunForAgent(run.agentId); + } + return cancelled; + } finally { + stopOwnership?.release(); + } + } + + async function cancelActiveForAgentInternal( + agentId: string, + reason = "Cancelled due to agent pause", + errorCode = "cancelled", + ) { + const agent = await getAgent(agentId); + const runs = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + ), + ); + + for (const run of runs) { + const stopOwnership = + run.runtimeMode !== "native" + ? captureAdapterStopOwnership(run.id) + : undefined; + try { + if (stopOwnership?.control) { + await cancelRunInternal(run.id, reason, { errorCode }); + continue; + } + if (run.runtimeMode === "native") { + await db.update(heartbeatRuns).set({ resultJson: + sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`, + }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + }); + } + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : parseObject(run.resultJson); + await setRunStatus(run.id, "cancelled", { + finishedAt: new Date(), + error: reason, + errorCode, + resultJson: { + ...persistedCancellationResult, + ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: persistedCancellationResult, errorCode, errorMessage: reason, + }) : {}), + cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason), + }, + }); + + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: new Date(), + error: reason, + }); + + const running = runningProcesses.get(run.id); + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid, + processGroupId: running.processGroupId, + graceMs: Math.max(1, running.graceSec) * 1000, + }); + } + runningProcesses.delete(run.id); + await releaseIssueExecutionAndPromote(run); + } finally { + stopOwnership?.release(); + } + } + + return runs.length; + } + + async function cancelPendingWakeupsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + if (uniqueAgentIds.length === 0) return 0; + + const now = new Date(); + const wakeupIds = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + inArray(agentWakeupRequests.agentId, uniqueAgentIds), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + + if (wakeupIds.length === 0) return 0; + + await db + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + updatedAt: now, + }) + .where(inArray(agentWakeupRequests.id, wakeupIds)); + + return wakeupIds.length; + } + + async function cancelInvocationsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + let runsCancelled = 0; + for (const agentId of uniqueAgentIds) { + runsCancelled += await cancelActiveForAgentInternal(agentId, reason); + } + const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( + uniqueAgentIds, + reason, + ); + return { + agentIds: uniqueAgentIds, + runsCancelled, + wakeupsCancelled, + }; + } + + async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { + const runIds = scope.scopeType === "project" + ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) + : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, scope.companyId), + scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, + scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + )).then((rows) => rows.map((row) => row.id)); + for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); + await cancelPendingWakeupsForBudgetScope(scope); + } + + return { + cancelBudgetScopeWork, + releaseIssueExecutionAndPromote, + releaseEnvironmentLeasesForRun, + acknowledgeRemoteStop, + resumeRemoteStopComments, + cancelRunInternal, + resumeExecutionWaitComments, + resumeQueuedCommentInterrupt, + resumeSavedLegacyComments, + cancelActiveForAgentInternal, + cancelInvocationsForAgentsInternal, + }; +} From 835a022936ddc7a152b9feab0c0420d02249a6d1 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:01:26 -0500 Subject: [PATCH 2/2] perf(server): wake delivery queues from transaction-aware work signals (#15625) ## Thinking Path > - Paperclip manages AI-agent work and its delivery to users and agents. > - Five delivery queues scan the database even when empty. > - Each queue already has durable rows; empty polling wastes idle hosting capacity. > - Producers can register intent inside their existing database transaction. > - Central transaction tracking can wake consumers after the outer commit without extra caller wrappers. > - A lost response needs reconciliation against PostgreSQL before an empty queue is safe to leave idle. > - This change uses one scheduler for outstanding work and lets settled, empty queues go quiet. ## Linked Issues or Issue Description Supersedes the closed #15614. Related idle-safety work: #15522 and #15599. **What existing behavior does this improve?** Delivery scheduling for feedback exports, chat completions, connection continuations, question answers, and tool-action receipts. **Current behavior** Feedback scans every five seconds. Four delivery queues scan on the heartbeat interval, normally 30 seconds. Startup and some event paths also scan them. **Proposed behavior** Each producer awaits one intent registration before its queue write. The existing `createDb` transaction boundary tracks nested savepoints and wakes the consumer after outer settlement. Startup scans restore durable work. Pending rows, failures, and unresolved transactions retain retry deadlines. Empty, settled queues have no timer or recurring scan. **Reason and benefit** Normal delivery starts after commit. Central tracking removes caller-specific post-commit plumbing. PostgreSQL transaction status resolves lost responses without new tables or a permanent uncertainty latch. **Breaking changes** No API or schema changes. The notification scope is one DB owner and its dedicated child pools. Direct SQL, separate roots, and other processes require an explicit wake/recovery integration. This path requires PostgreSQL 14+ transaction-information functions; embedded PostgreSQL uses 18. ## What Changed - Add one named-deadline scheduler and app-owned coordinator for five existing queues. - Instrument `createDb().transaction()` centrally, including nested savepoints. Dedicated child connections share their owner's signal scope. - Register work before the five queue insert paths. The first registration obtains the outer XID; unrelated transactions issue no extra SQL. Tool receipt insertion gains a small transaction around its existing insert. - Query `pg_xact_status` only for rejected, tracked transactions. Probe before scanning the queue. Retain the idle hold while PostgreSQL still reports an in-progress transaction or the probe fails. Release it after settlement and reconciliation. - Remove unconditional delivery recovery scans and caller-specific completion post-commit actions. Retain the existing task-scoped completion activity fast path and durable claims. - Coalesce wakes without postponing an earlier deadline. Retry outstanding rows and failures; suppress dispatch during idle drain while allowing transaction and queue reconciliation; suppress both during warm standby. Cancel deadlines and await active delivery sweeps on shutdown. - Serialize feedback flushes. Vote routes return after saving. Uploads have a 30-second deadline and shutdown cancellation; unfinished exports remain recoverable. - Document the writer contract and update the dated sleep inventory. ## Verification - Final head: `a4609956841107ad60c4cb50fe2acb05636b1363`. - Passed: all final-head hosted checks, with 54 successes and two intentional skips. The full test matrix, typecheck, build, canary, and aggregate verification passed in [run 37940948440](https://github.com/paperclipai/paperclip/actions/runs/37940948440). - Passed: Greptile 5/5 on the same head, with a successful check run and zero unresolved review threads. The branch is mergeable. - Passed locally after the final rebase: 50 coordinator, scheduler, and startup tests; server typecheck; server build. - Passed locally before the final import-only rebase: repo-wide typecheck and build; 237 PostgreSQL producer/delivery and database signal tests; feedback, native-question, and idle-safety regression tests. - Real PostgreSQL tests hold transactions open after an injected client response loss, then commit or abort. They verify that an empty scan cannot clear an in-progress transaction. Another test terminates a real backend and verifies recovery without replay. - Coordinator tests cover an hour with no empty-queue timers, pending/error retries, worker replacement, concurrent writes, earliest-deadline preservation, rollback during idle drain, committed work during drain, standby, and shutdown. - Local `pnpm test:run` was started and stopped after embedded PostgreSQL startup failures appeared. It did not finish and is not reported as a pass. Later local database reruns had skipped suites; those skips are not claimed as verification. The earlier PostgreSQL tests did execute and pass, and the final hosted full matrix passed. ## Risks - One XID query is added per transaction that registers delivery work. Nested transactions share that ID. Registration must be awaited before writing; new enqueue paths must follow this contract. - Work signals are local to a DB owner and its dedicated child pools. Independent roots, direct SQL, or other processes are not observed. Startup scans recover already committed rows, but do not fence late transactions from a previous process. Cross-process ownership and database failover require separate work. - A database outage or genuinely unresolved transaction keeps an idle hold until status can be reconciled. No timer clears uncertainty by assumption. - These changes remove five empty polling loops. They do not implement whole-instance sleep, database teardown, or an external waker. The earliest deadline covers only the registered queues. - Waiting tool reviews retain their existing retry cadence until their receipts can be delivered. - Feedback vote responses no longer wait for the remote upload. Sharing consent and the saved vote response are unchanged. - Real hosting-provider sleep and cost savings have not been measured. ## Model Used OpenAI Codex, GPT-6. Used reasoning, repository inspection, code editing, and command execution. The runtime did not expose the exact model revision or context window. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/DATABASE.md | 43 + ...-instance-sleep-and-scheduler-inventory.md | 867 ++++++++++++++++++ packages/db/src/client.ts | 11 +- packages/db/src/index.ts | 2 + packages/db/src/work-signals.test.ts | 165 ++++ packages/db/src/work-signals.ts | 109 +++ .../chat-completion-delivery.test.ts | 55 +- .../connection-intents-service.test.ts | 6 + .../delivery-work-coordinator.test.ts | 192 ++++ server/src/__tests__/feedback-service.test.ts | 35 +- .../__tests__/feedback-share-client.test.ts | 42 +- .../__tests__/issue-feedback-routes.test.ts | 21 +- .../issue-thread-interactions-service.test.ts | 6 + .../server-startup-feedback-export.test.ts | 29 +- .../__tests__/tool-gateway-service.test.ts | 7 + server/src/__tests__/work-scheduler.test.ts | 47 + server/src/app.ts | 52 +- server/src/index.ts | 58 +- server/src/routes/issues.ts | 26 +- .../src/services/chat-completion-delivery.ts | 18 +- .../services/connection-intent-delivery.ts | 6 +- .../src/services/delivery-work-coordinator.ts | 100 ++ .../services/delivery-work-notifications.ts | 22 + server/src/services/feedback-share-client.ts | 12 +- server/src/services/feedback.ts | 30 +- .../src/services/issue-thread-interactions.ts | 3 + .../services/question-response-delivery.ts | 6 +- server/src/services/tool-action-delivery.ts | 6 + server/src/services/tool-action-review.ts | 2 + server/src/services/tool-gateway.ts | 22 +- server/src/services/work-scheduler.ts | 45 + 31 files changed, 1908 insertions(+), 137 deletions(-) create mode 100644 doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md create mode 100644 packages/db/src/work-signals.test.ts create mode 100644 packages/db/src/work-signals.ts create mode 100644 server/src/__tests__/delivery-work-coordinator.test.ts create mode 100644 server/src/__tests__/work-scheduler.test.ts create mode 100644 server/src/services/delivery-work-coordinator.ts create mode 100644 server/src/services/delivery-work-notifications.ts create mode 100644 server/src/services/work-scheduler.ts diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 7799ead39ed..a90cd328622 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -681,3 +681,46 @@ and are atomically deleted before code exchange. The `code_verifier` column hold this non-secret binding for this namespace; Slack bot installation does not use PKCE. Removal and manual recovery invalidate outstanding attempts under the same credential-mutation lease used by configuration. + + +## Transaction-aware delivery work signals + +Five existing delivery queues use process-local work signals to avoid empty +polling. Register work **before** writing a queue row, on that row's transaction: + +```ts +await db.transaction(async tx => { + await signalDatabaseWork(tx, "queue-topic"); + await tx.insert(existingQueueTable).values(row); +}); +``` + +`createDb` instruments transaction callbacks and nested savepoints. Subscribers +receive intent immediately and settlement only after the outer transaction +finishes. Caller-owned transactions therefore need no extra post-commit wrapper. +Dedicated clients from `withDedicatedDbConnection` share the owner's signal +scope. No schema changes, triggers, dedicated listener connection, or periodic +queries are installed by this API. + +The first registered write in a transaction fetches `pg_current_xact_id()`. +Awaiting registration is required: a failure at that point must prevent the +queue write. Unrelated transactions add no queries. If the transaction rejects, +the coordinator probes `pg_xact_status(xid)` through the root pool before scanning +the queue. An in-progress transaction remains unresolved even when the queue +currently looks empty. Committed/aborted results permit reconciliation; NULL +means PostgreSQL has discarded an old, no-longer-active transaction's outcome. +The existing durable queue supplies the work in either case. Probe failures +retain the idle hold and schedule another attempt. Work signals never replay +queries or convert a failed database operation into success. + +This uses PostgreSQL's transaction-information functions (this path requires +PostgreSQL 14+; embedded PostgreSQL uses 18). See the +[PostgreSQL transaction information documentation](https://www.postgresql.org/docs/18/functions-info.html#FUNCTIONS-PG-SNAPSHOT). +The process-local intent survives coordinator replacement, but not process exit. +Startup scans recover committed queue rows. Independent DB clients, other +processes, late transactions from an old process, and database failover need an +explicit ownership/wake protocol; these signals are not cross-process messaging. +Use the same root client for in-process writers. New queue insertion paths must +register before writing, or they can remain unseen until the next startup or +another notification. Tests should verify both the producer and its outer +transaction boundary. diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md new file mode 100644 index 00000000000..cc7df120edd --- /dev/null +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -0,0 +1,867 @@ +# Instance sleep and scheduler inventory + +Date: 2026-10-08 + +Status: Inventory with first delivery-coordinator implementation; full idle sleep remains open + +Code baseline: `0ac194450` + +Paperclip should release idle database connections and allow inactive customer +instances to sleep without losing scheduled work, incoming events, accounting, +or recovery. Thousands of individually hosted instances make empty polling a +material cost even when each query is cheap. + +**Architectural direction: triage each loop before choosing its replacement.** +Work caused by a committed change or incoming event should run from that event. +Work caused by the passage of time belongs in one in-process, instance-wide +scheduler. A later external-waker adapter publishes only the stack's next wake +time. Routine definitions and execution decisions remain inside the stack. + +Immediate delivery does not have to pass through the scheduler. Its delayed +retry does. The same service can contain both categories; classify its work +paths rather than treating its current polling loop as one indivisible job. + +The database already closes idle pooled connections after 60 seconds and opens +new connections on demand. The primary work is eliminating unnecessary queries +and coordinating background work, connections, and hosting sleep. + +## First slice: transaction-aware delivery notifications and shared scheduling + +Updated: 2026-10-09. This implementation covers five existing durable queues: + +- [x] JOB-04: feedback exports register intent before the export write. +- [x] JOB-22: completion receipts register intent inside their transaction, including caller-owned and nested transactions. No extra post-commit action is needed from callers. +- [x] JOB-23: connection continuations register intent before enqueue. +- [x] JOB-25: question answers register intent before enqueue. +- [x] JOB-24: tool receipts and review resolution register intent before enqueue. +- [x] `createDb` centrally defers delivery wakes until the outer transaction settles. Dedicated child pools share the owning client's signal scope. +- [x] One scheduler owns pending-work and failure retries. Empty, settled queues have no recurring deadline or database scan. +- [x] Unresolved local writes prevent idle admission. Recovery probes PostgreSQL's transaction status before scanning the existing queue and clears the hold once both are settled. +- [x] Tests cover nested commits, rollback, late commit/abort after a lost client response, concurrent writes during a sweep, worker replacement, restart recovery, drain/standby, and shutdown. +- [ ] Prove the complete instance writer-admission and idle/resume boundary across all other work. +- [ ] Migrate other actual scheduled work and expose the whole instance's earliest deadline to an external waker. +- [ ] Verify actual app and database sleep on the hosting providers. + +Existing delivery rows remain authoritative; no tables or migrations are added. +A producer adds one awaited `notifyDeliveryWork(tx, queue)` before its write. +`createDb().transaction()` and nested savepoints are instrumented centrally. The +first opted-in write fetches the outer transaction's XID; unrelated transactions +add no SQL. A rejected transaction with a known XID is reconciled with +`pg_xact_status` before the queue scan. An empty scan cannot discharge an +in-progress transaction. Committed, aborted, or aged-out XIDs can be discharged +after scanning. A failed status query retains the hold and a retry deadline. + +The scheduler retains startup sweeps, retry deadlines while durable work exists, +and retries after failures. Idle drain permits transaction and queue checks but +does not start deliveries; warm standby suppresses both. There is no +unconditional 60-second recovery pass. +Feedback flushes remain serialized and cancellable, with a 30-second upload +limit. Vote routes return after saving rather than waiting for upload. Tool +receipts awaiting review or execution still retain their retry cadence. + +**Scope:** notifications and unresolved-XID tracking are process-local to a +`createDb` owner and its dedicated child pools. They do not observe direct SQL, +an independently created client, or another process. Such writers need an +explicit wake/reconciliation integration before this mechanism can support +them. Startup scans recover already committed durable rows; they do not prove +that transactions still running in an old process have finished. Cross-process +handoff/fencing is separate work, as is PostgreSQL failover during uncertainty. + +This removes these five empty polling loops; it does not establish whole-stack +sleep. `nextWakeAt` covers only registered queues. Database connection teardown, +other background loops, full idle reconciliation, and external wake registration +remain separate work. + +Implementation: `packages/db/src/work-signals.ts`, +`server/src/services/work-scheduler.ts`, `delivery-work-coordinator.ts`, and +`delivery-work-notifications.ts`. + +## Using this checklist + +Items have stable IDs so they can become separate issues or PRs. All boxes start +unchecked: an existing mechanism or a recommendation is not proof that the +sleep journey has passed. Mark an item complete only after its completion +criteria have evidence, and append the issue/PR and verification result to it. +Record deferrals with a reason and the capability that must remain awake. + +Source links identify files in the audited revision. Function names and stated +cadences are navigation aids; recheck them when implementing a later revision. +Cadences describe ordinary claimed-instance operation. Warm-standby, idle-drain, +configuration, or feature gates can suppress individual paths. This inventory +does not establish the actual deployed fleet configuration. + +## Triage before implementation + +Use this question for every operation: **If no new request or event arrives, +does this operation still need to happen at a particular future time?** + +- **Yes: clock-driven.** Register that deadline with the internal scheduler. + Examples are a routine at 09:00, a retry after backoff, and a session expiry. +- **No: event-driven or on demand.** Invoke it from the owning committed mutation, + verified external event, or user request. An empty queue is not a scheduled + task. Examples are delivering a newly committed answer and refreshing a UI + after an issue changes. + +Many existing sweeps combine both. Separate their immediate dispatch from their +retry/timeout/recovery paths. Persistent transports and active-operation +keepalives are lifecycle requirements, not idle business schedules; their +ownership and shutdown need separate treatment. + +The tables below are a first-pass mechanism triage grounded in the inventory. +They select where to investigate first, not permission to delete a recovery +path. Detailed checklist items remain the implementation and verification record. + +### Work that genuinely needs a clock + +| Checklist items | Reason time matters | What belongs in the scheduler | +| --- | --- | --- | +| JOB-08 Agent timer heartbeats and issue monitors | The user explicitly requested future checks. | The next enabled heartbeat/monitor deadline; none when unconfigured. | +| JOB-09 Routine cron triggers | A routine must run even without new incoming activity. | Existing `nextRunAt`, retaining catch-up and timezone semantics. | +| JOB-07 Plugin scheduled jobs | A plugin may declare real recurring work. | Declared job deadlines; plugin load/configuration events update registration. | +| JOB-29 Backups | A backup guarantee has a time requirement. | The next backup if this instance owns it; otherwise the explicit external backup owner. | +| JOB-18, JOB-19 Login and setup expiry | An abandoned session/resource must expire even if nobody returns. | Actual expiry and failed-cleanup retry times, only while relevant sessions exist. | +| JOB-11, JOB-21 Decision and secret/token expiry | Decision deadlines and some deletion/retention obligations are time-based. | Required expiry/scrub deadlines. Check validity synchronously on use regardless of whether physical cleanup ran. | +| JOB-10 Decision retention | An unchanged item can become old enough to archive. | The next actual retention deadline; immediate change handling and notification delivery are event-driven. | +| JOB-12, JOB-13 External objects and status cards | Some configured freshness/update policies require future work. | Only time-based refresh/evaluation deadlines; reactive updates use events. | +| JOB-27, JOB-28 Retries and watchdogs | A retry delay or absence of expected progress can itself require action. | Concrete retry, lease-expiry, monitor, or watchdog deadlines for existing work. | + +### Work whose normal path should stop polling + +| Checklist items | Preferred normal trigger | Clock-driven remainder or qualification | +| --- | --- | --- | +| JOB-04 Feedback exports | Committed export enqueue. | Backoff after delivery failure and durable recovery of outstanding exports. No empty five-second scan. | +| JOB-22 Chat completion delivery | Committed task completion/update. An existing postcommit fast path is already present. | Delivery retries and recovery; the current fast path is best-effort, so prove replacement reliability before removing its sweep. | +| JOB-23 Connection continuation delivery | Committed connection availability/change and pending continuation enqueue. | Failed-delivery retry/claim recovery. | +| JOB-25 Question response delivery | Committed user answer. | Retry/claim expiry while delivery is outstanding. | +| JOB-24 Tool action reviews and deliveries | Committed approval/rejection or action enqueue. | Review expiry and delivery retries. | +| JOB-26 Cost accounting | Arrival or durable capture of a usage receipt/result. | Recovery of unresolved receipts/reservations and failed writes. Accounting debt cannot be dropped. | +| JOB-05 MCP and Dot events | Committed source event or subscription change. | Delivery retries and required expiry cleanup. | +| JOB-01 Chat reconciliation | Incoming message, committed publication, or endpoint configuration change. | Split its lanes: delivery retry and missed-event recovery are temporal; socket ownership is NET work. | +| JOB-02 Email reconciliation | Verified webhook, outgoing send, or endpoint configuration change. | Retry and provider catch-up guarantees; socket reception and catch-up are NET-01/NET-02. | +| JOB-03 Browser-use cleanup | Browser/session lifecycle change. | Register actual idle/expiry/reconciliation deadlines while sessions exist; active leases remain protected. | +| JOB-06 Execution-control reconciliation | Execution state transitions and committed control requests. | Timeout, lease-expiry, abandoned-operation, and failed-delivery recovery. Split the lanes before changing cadence. | +| JOB-14 GitHub events and continuity | Verified webhook and connection lifecycle change where supported. | Credential deadlines; bounded polling only for providers/events without adequate push or replay guarantees. | +| JOB-15 Merge confirmations | Verified merge event for an outstanding candidate. | Bounded fallback only while candidates exist and webhook/replay coverage is insufficient. | +| JOB-16 Terminal workspaces | Committed terminal task/tree transition. | Cooldown, failed-cleanup retry, and recovery of outstanding cleanup. | +| JOB-17 Sandbox cleanup | Resource release or failed acquire/release. | Retry/backoff and provider/fleet orphan recovery. | +| JOB-20 Connection health | Use of the connection or configuration change. | Proactive checks only where a defined freshness guarantee requires them. | +| JOB-10, JOB-11 Decision effects and notifications | Committed decision/relevant source mutation. | Expiry/retention stays in the clock-driven category. | +| JOB-12, JOB-13 Reactive external objects/cards | Verified provider event or relevant local mutation. | Configured time-based refresh stays in the clock-driven category. | +| JOB-27, JOB-28 Queued runs, assignments, and dependencies | Committed enqueue, assignment, terminal result, or dependency transition. | Time-based recovery/watchdogs stay in the clock-driven category. | +| AUX-01, AUX-03 Telemetry and plugin log flushing | First buffered entry activates a bounded flush. | Batch delay and failed-delivery retry while nonempty; no empty heartbeat. | +| AUX-04 Import spools and runtime status expiry | Spool creation/completion, startup, or access to an expiring in-memory value. | Schedule only required file-retention/cleanup deadlines. Lazy expiration can handle noncontractual memory cleanup. | +| UI-01, UI-02 Stable UI data | Existing live events, initial read, foreground return, and reconnect reconciliation. | Preserve bounded fallback during a genuine realtime outage or active operation. Reconcile subscription gaps because live events are not replayed. | +| UI-03, UI-04, UI-06 Recovery, app updates, and setup | Foreground/online/user activity and bounded setup state. | Client-side retry/update deadlines where needed; these do not become tenant server jobs. | + +JOB-30 is removal of the aggregate polling wrapper after its constituents are +handled, not a new scheduled job. DB-01–DB-03 concern the connection pool; +DB-04/DB-05 concern probes. NET-01–NET-07, UI-05, and AUX-05 concern transport or +plugin lifetime. AUX-02/AUX-06 require checking actual exporter/SDK traffic. +SCH, WAKE, SAFE, HOST, and TEST items are infrastructure, policy, or verification +work rather than business jobs. This keeps every inventory item accounted for +without making the scheduler responsible for every source of activity. + +### Qualification before removing a poll + +- [ ] **TRI-01 — Identify the real trigger and every producer.** For each + event-driven candidate, list the committed mutations/external events that + create work, including alternate API paths and recovery writers. **Complete + when:** every supported producer has an owned dispatch path and the service + needs no scan to discover normal new work. + +- [ ] **TRI-02 — Separate first delivery from delayed recovery.** Record which + paths are immediate, which have a concrete due time, and which depend on an + external provider's push capabilities. **Complete when:** each migrated lane + has an explicit trigger and its remaining deadlines are enumerated. + +- [ ] **TRI-03 — Prove durable delivery before retiring its backstop.** A + best-effort callback after commit is not a durable queue consumer. Cover + commit-before-signal crashes, lost signals while the process stays alive, + handler failures, concurrent producers, and restart. **Complete when:** an + outstanding record cannot become invisible indefinitely. Retain a bounded, + explicitly justified reconciliation path until this is proven; document its + wake cost rather than assuming startup recovery covers every failure. + +- [ ] **TRI-04 — Order work by avoidable idle traffic and migration risk.** + Record expected idle SQL/network reduction and the smallest focused proof for + the next item. **Complete when:** the next PR has one clear boundary and does + not require the entire scheduler or all connector lifecycles to be finished. + +### Initial implementation shortlist + +1. **DB-04/DB-05: health and probe separation.** A self-contained source of + avoidable demand, independent of building a scheduler. Verify actual callers + so the new endpoint is used. +2. **JOB-04: event-driven feedback exports.** A contained first dispatch + migration. Triage its enqueue, failed-export behavior, and recovery first; + put only actual retry/batch deadlines in scheduling infrastructure. +3. **JOB-22/JOB-25: completion and answer delivery.** Reuse existing postcommit + paths where possible, after proving that durable delivery can replace the + sweep rather than merely supplement it. +4. **JOB-01/JOB-02/JOB-03: frequent empty chat, email, and browser scans.** Large + baseline benefit, but split the component lanes and preserve configured + transports, expiry, and recovery. Resource absence must be invalidated when + a new resource is committed. +5. **SCH with JOB-09: routine scheduling as the first true clock-driven slice.** + Existing `nextRunAt` makes the real scheduler requirement concrete. Add other + temporal paths and then the external-waker adapter. + +This order supersedes starting by routing the feedback polling loop through a +generic scheduler. Event-driven improvements can ship before that scheduler is +complete. The resulting service still needs an owned way to retry failures. + +## Provider behavior and cost model + +| Layer | Documented behavior | Consequence | +| --- | --- | --- | +| Neon | Normally suspends after five minutes without active queries. Idle-in-transaction connections count as active; ordinary idle connections do not necessarily block suspension. New connection requests can reset inactivity. | Stop empty SQL polling and connection churn; do not force-close active transactions. | +| Railway | Detailed Serverless documentation specifies five minutes without outbound packets, sampled so actual sleep is roughly 5–10 minutes. Private-network traffic and responses count. Enabling Serverless requires a deployment to affect the container. | Every recurring source of packets matters. Even a DB-free endpoint can keep the app awake when repeatedly called externally. | +| Unikraft | `on` waits for connections/HTTP requests to finish; `idle` can suspend inactive TCP connections. Active HTTP requests prevent suspension. Stateful mode resumes a snapshot. | Quiesce connections deliberately and explicitly protect background work from premature suspension. | + +Provider sources: [Neon compute lifecycle](https://neon.com/docs/introduction/compute-lifecycle), +[Neon compute management](https://neon.com/docs/manage/endpoints/), +[Railway Serverless](https://docs.railway.com/deployments/serverless), +[Unikraft scale to zero](https://unikraft.com/docs/features/scale-to-zero). + +Neon usage is reported in CU-seconds. The transcript's five-minute concern is +primarily an idle-timeout cost, not a universal billing quantum. With a +five-minute timeout, a trivial hourly check creates roughly an 8.3% active-time +floor before useful work; checking every five minutes can leave compute +continuously active. This illustration assumes the default timeout and excludes +other traffic, processing time, and storage charges. Verify actual endpoint +settings and the applicable plan. See [Neon usage calculations](https://neon.com/docs/introduction/usage-calculations). + +## 1. Internal scheduler for true deadlines + +- [ ] **SCH-01 — Establish the instance-wide scheduler boundary.** Route + application-owned delayed jobs, recurring jobs, retry deadlines, and required + expiry/cleanup/maintenance deadlines through one scheduler shared across all + companies. Immediate event/queue dispatch remains in its owning service; + running-work tracking is shared with sleep admission. Keep domain handlers and + their authorization/atomic claims in their existing services. **Complete + when:** one scheduling interface supports true temporal obligations, company + scope remains explicit, and self-hosted operation requires no external service. + +- [ ] **SCH-02 — Model work and deadlines explicitly.** Track a stable job/lane + identity, company scope where applicable, due time, pending/running state, + retry policy, and whether running work blocks sleep. Accept running-work + information from event-driven handlers without forcing their dispatch through + the scheduler. Distinguish scheduled + work, immediate queued work, waiting for an external event, and passive + configuration/history. **Complete when:** an empty lane has no recurring SQL + check, and the scheduler can explain outstanding work and the earliest due + time without querying the database on every timer tick. + +- [ ] **SCH-03 — Reconstruct state and handle committed changes.** Seed the + scheduler from existing durable domain records on startup; use committed + mutations to register, reschedule, cancel, or activate work. In-memory hints + cannot be the sole record of an obligation. Define how all supported writers + invalidate hints; initially retain the existing single-process boundary. + **Complete when:** restart and a crash between commit and notification recover + work, and creating an earlier deadline updates the scheduler immediately. + +- [ ] **SCH-04 — Implement local scheduling and catch-up.** Arm the nearest + necessary local timer and run due handlers with bounded concurrency and + per-lane overlap protection. Preserve each domain's timezone, missed-run, + coalescing, priority, and retry semantics. **Complete when:** delayed callbacks, + clock changes, restart, and duplicate activation cannot silently lose work or + bypass existing execution claims. + +- [ ] **SCH-05 — Expose an aggregate next wake time.** Compute the earliest + required deadline across the whole instance, including retries, cleanup, + accounting, and maintenance. Represent no scheduled work as no alarm; represent + unknown obligations as a sleep blocker. **Complete when:** adding, moving, + removing, or completing the earliest item correctly changes the aggregate. + +- [ ] **SCH-06 — Integrate running-work ownership and drain.** Reuse + [task admission](../../server/src/services/task-admission.ts), + [idle admission](../../server/src/middleware/idle-admission.ts), and the + [existing sleep protocol](../idle-sleep-safety.md). Track handler completion + even after an HTTP response or client disconnect. **Complete when:** draining + prevents new scheduled admissions and waits for accepted work without losing + fire-and-forget work or mutations racing the final check. + +- [ ] **SCH-07 — Classify transport and SDK timer exceptions.** Business + scheduling belongs to the scheduler. Protocol pings, socket deadlines, and + active-operation lease renewals can remain local to their transports, but + must have explicit start/stop ownership and participate in sleep blocking. + Browser timers use a separate foreground/activity policy. **Complete when:** + exceptions are inventoried and owned; no hidden independent timer can renew + an idle connector indefinitely after its scheduler lane is stopped. + +- [ ] **SCH-08 — Make scheduler migration incremental and observable.** Migrate + each inventory item below independently. Report pending work, next deadline, + active holds, and unknown/unmigrated blockers through local diagnostics. + Avoid periodic SQL or remote reporting merely to observe inactivity. + **Complete when:** an operator can explain why a stack is awake and a migrated + lane can be rolled back without duplicate processing. Preserve the telemetry + review rules if any first-party event contract changes. + +## 2. External waker adapter later + +The minimal external model is `(stack, nextWakeTime)`. Operational delivery may +also carry a generation/idempotency value and delivery status. It must not +require copying every routine, agent configuration, or permission into Cloud. +The internal scheduler remains authoritative about what to execute. + +```mermaid +flowchart LR + D[Temporal obligations and retry deadlines] --> S[Instance scheduler] + S --> L[Local timer while awake] + S -->|stack, nextWakeTime, generation| W[External waker adapter] + W --> G[Wake-aware ingress or provider wake] + U[User request or webhook] --> G + G -->|Reconcile due deadlines on wake| S + G -->|Verified request or event| H + L --> H[Existing domain handlers] + S --> H +``` + +- [ ] **WAKE-01 — Define the adapter and local-only behavior.** Support + setting/replacing and cancelling the next alarm. No external adapter is + required to use the internal scheduler. **Complete when:** timed work remains + local and the instance remains awake if no durable external wake is available; + introducing the internal scheduler alone never authorizes sleeping through a + deadline. + +- [ ] **WAKE-02 — Make alarm publication durable and acknowledge it before sleep.** + Persist publication intent/generation or use an equivalent recoverable + handshake. Recheck that the acknowledged alarm still covers the earliest + deadline before sleeping. **Complete when:** failure, lost responses, and a + concurrently moved-earlier deadline keep the instance awake until covered; + local scheduling continues during external-waker outages. + +- [ ] **WAKE-03 — Handle duplicate, stale, delayed, and missed wake delivery.** + Fence stale generations, retry failed delivery, reconcile due work after + wake, and retain domain-level atomic claims. **Complete when:** a repeated + doorbell does not duplicate work, and missed deadlines follow each domain's + explicit catch-up policy. Do not claim exactly-once network delivery. + +- [ ] **WAKE-04 — Serialize ingress, sleep, deploy, and wake.** Buffer or retry + incoming requests while draining; authenticate external control and preserve + webhook verification and acknowledgement deadlines. The host must bind its + final stop decision to the correct process/provider generation. **Complete + when:** requests arriving during sleep entry are neither lost nor admitted + into a process being stopped. Multi-replica support requires shared fencing. + +- [ ] **WAKE-05 — Implement and qualify provider adapters.** Evaluate Unikraft + scheduled `start` operations and a shared durable delayed-delivery mechanism + for Railway. Verify that provider resume actually invokes due-work processing, + including stateful resume without process startup. **Complete when:** an + isolated instance wakes for its earliest deadline and no waker polls sleeping + tenant databases. See [Unikraft scheduled wake-ups](https://unikraft.com/docs/features/cron-jobs). + +- [ ] **WAKE-06 — Protect active background execution on the host.** Connect + active-work holds to the provider's sleep controls. Unikraft exposes a + reference-counted disable mechanism; account for its signal delay. **Complete + when:** startup, transactions, agent control, finalization, and cleanup cannot + be suspended halfway through accepted work. See [Unikraft application sleep controls](https://unikraft.com/docs/tutorials/scale-to-zero-triggers). + +- [ ] **WAKE-07 — Validate wake concentration and cost.** Capacity-test concurrent + scheduled wakes, apply bounded dispatch, and use jitter only where semantics + permit. **Complete when:** scheduler backlog/delivery delay is observable and + a midnight burst does not silently skip customer jobs. Avoid periodically + waking all tenants just to discover whether any work exists. + +## 3. Database and health + +- [ ] **DB-01 — Qualify the existing idle pool behavior.** + [client.ts](../../packages/db/src/client.ts) already defaults + `DATABASE_IDLE_TIMEOUT_SECONDS` to 60; `0` disables reaping. Keep the existing + Drizzle handle and lazy reconnection; evaluate 30 seconds after pollers become + quiet. **Complete when:** idle connections disappear and a subsequent read, + write, and transaction work after actual provider suspension without per-route + lifecycle code. Existing tests are in + [client.test.ts](../../packages/db/src/client.test.ts). + +- [ ] **DB-02 — Preserve disconnect and transaction safety.** `.end()` permanently + shuts down the pool and must not become an idle operation. Reconnection does + not authorize replaying an interrupted write. **Complete when:** existing + transaction isolation after disconnect and explicitly idempotent retry rules + still hold; ambiguous commits are not blindly replayed. See + [database retry behavior](../DATABASE.md#connection-loss-and-retries). + +- [ ] **DB-03 — Cover every client and socket.** Better Auth shares the main DB; + the optional plugin migration pool uses the same idle configuration. Utility, + backup, and dedicated advisory-lock connections are bounded work. The audited + postgres.js version has a 60-second TCP keepalive; keepalive is not SQL. + **Complete when:** all pools can reach zero when idle, reserved/active work is + protected, and no new persistent LISTEN or health-ping connection is introduced + as a substitute for polling. Sources: + [DB client](../../packages/db/src/client.ts), + [auth adapter](../../server/src/auth/better-auth.ts), + [backup library](../../packages/db/src/backup-lib.ts). + +- [ ] **DB-04 — Split process liveness from DB readiness.** Claimed-instance + [health](../../server/src/routes/health.ts) executes `SELECT 1`; auth and deep + checks may perform additional queries. Install cheap liveness before session + and actor resolution. **Complete when:** normal liveness produces no SQL, + explicit deep readiness still tests DB health, and signed claim/bootstrap + mutations retain their existing validation and tracking. + +- [ ] **DB-05 — Audit external probes and configured endpoints.** Inspect the + fleet gateway, uptime monitors, deployment checks, DB pool settings, and Neon + suspend configuration. The [ECS example](../../docker/ecs-task-definition.json) + probes every 30 seconds; it is not evidence of Railway fleet behavior. + **Complete when:** sleeping is an expected state, repeated checks do not wake + tenants or reconnect to Neon, and readiness errors remain distinguishable + from intentional sleep. Railway's built-in health checks are deployment-only: + [provider documentation](https://docs.railway.com/deployments/healthchecks). + +## 4. Server scheduling migration inventory + +Apply the triage above to each operation within these services. Remove polling +from event-driven normal paths and route only real deadlines through the internal +scheduler. Preserve durable delivery, startup recovery, and domain-level claims. +Longer polling intervals alone do not complete an item. + +- [ ] **JOB-01 — Chat reconciliation.** Current: every **1s**, with multiple + runtime/delivery/publication lanes and SQL even without connectors. Change: + disarm empty lanes and activate on committed events or deadlines. **Complete + when:** a tenant with no chat obligations issues no chat SQL and new inbound, + publication, and recovery work still runs. Sources: + [app.ts](../../server/src/app.ts), + [chat-channels.ts](../../server/src/services/chat-channels.ts). + +- [ ] **JOB-02 — Email reconciliation.** Current: every **1s**, querying active + endpoints even when none exist. Change: initialize resource/pending state and + update on endpoint changes, webhook receipt, sends, and retries. **Complete + when:** empty mail state produces no polling SQL and new work activates the + worker without restart. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **JOB-03 — Browser-use cleanup.** Current: every **3s** with due-session + queries even when empty. Change: register actual session reconciliation/expiry + deadlines. **Complete when:** no sessions means no queries, while active leases, + expiry, and failed cleanup remain recoverable. Sources: + [app.ts](../../server/src/app.ts), + [browser-use.ts](../../server/src/services/browser-use.ts). + +- [ ] **JOB-04 — Feedback exports.** Current: every **5s**, querying the queue + even without a configured share client. Change: committed enqueue signals and + durable retries. **Complete when:** empty exports are quiet and queued exports + survive restart and temporary delivery failure. Sources: + [app.ts](../../server/src/app.ts), + [feedback.ts](../../server/src/services/feedback.ts). + +- [ ] **JOB-05 — Public MCP and Dot event dispatch.** Current: **2s** per started + dispatcher, including SQL-backed enablement, expiry, and delivery scans. + Change: cache capability state and schedule delivery/retry/expiry work. + **Complete when:** disabled/empty dispatchers are DB-free and pending events + retain subscription scope, expiry, and replay safety. Source: + [events.ts](../../server/src/services/public-mcp/events.ts). + +- [ ] **JOB-06 — Execution-control reconciliation.** Current: independent + **15s** recovery/delivery lanes, including when the main heartbeat is disabled. + Change: activate lanes from durable pending work and deadlines. **Complete + when:** empty lanes are quiet and abandoned finalizations, replacements, + continuations, status/disposition delivery, and login expiry still recover. + Sources: [index.ts](../../server/src/index.ts), + [execution-control-deadline.ts](../../server/src/services/execution-control-deadline.ts). + +- [ ] **JOB-07 — Plugin job scheduling.** Current: **30s** SQL even without jobs. + Change: register nearest `nextRunAt` and signal registration/update/removal. + **Complete when:** empty plugins/jobs produce no scheduler SQL and due jobs + retain atomic ownership. The plugin job coordinator itself is event-driven, + not another periodic loop. Sources: + [plugin-job-scheduler.ts](../../server/src/services/plugin-job-scheduler.ts), + [plugin-job-coordinator.ts](../../server/src/services/plugin-job-coordinator.ts). + +- [ ] **JOB-08 — Agent timer heartbeats and issue monitors.** Current: **30s** + agent/monitor scans; agent-level disabled heartbeats still require discovery. + Change: register each relevant deadline and aggregate locally. **Complete + when:** no timer work means no queries, and due work preserves pause, budget, + authorization, and atomic wake admission. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `tickTimers`. + +- [ ] **JOB-09 — Routine cron triggers.** Current: **30s** due-trigger query. + Change: use existing `nextRunAt` in the internal scheduler. **Complete when:** + timezone, catch-up/coalescing, concurrency, project pause, edits, and deletion + behave correctly; external waking can be added without relocating routines. + Source: [routines.ts](../../server/src/services/routines.ts), `tickScheduledTriggers`. + +- [ ] **JOB-10 — Decision retention and notifications.** Current: every **30s**, + builds the complete attention feed for each active company before archiving + and delivering notifications. Change: relevant mutation signals, actual + retention deadlines, and batched maintenance. **Complete when:** unchanged + idle companies do not rebuild feeds and required archive/notification behavior + remains timely. Source: [index.ts](../../server/src/index.ts), `runRetentionSweep`. + +- [ ] **JOB-11 — Decision expiry and recovery.** Current: **30s** scans for stuck + effects, continuations, and TTLs. Change: schedule their actual deadlines and + signal committed effects. **Complete when:** expiration remains authoritative + at action time and interrupted effects recover without perpetual empty scans. + Source: [decisions.ts](../../server/src/services/decisions.ts), `sweepExpired`. + +- [ ] **JOB-12 — External-object refresh.** Current: **30s** scans, including a + DB feature check when disabled. Change: invalidate cached capability on settings + changes; register `nextRefreshAt`; use webhook signals where supported. + **Complete when:** disabled/empty state is quiet and promised refresh freshness + has an owned deadline. Sources: + [external-objects.ts](../../server/src/services/external-objects.ts), + [instance-settings.ts](../../server/src/services/instance-settings.ts). + +- [ ] **JOB-13 — Status-card evaluation.** Current: **30s**, including a DB + feature check when disabled. Change: register `nextEvalAt` and relevant event + invalidations. **Complete when:** no capability/due cards means no queries, + and evaluation preserves existing assignment and cancellation behavior. + Source: [status-cards.ts](../../server/src/services/status-cards.ts). + +- [ ] **JOB-14 — GitHub events and connection continuity.** Current: **30s** + connection scans; provider calls depend on configured/due work. Change: + webhook-first activation plus explicit poll fallback, credential, and retry + deadlines. **Complete when:** empty state is quiet and configured connections + retain their delivery and continuity guarantees. Source: + [index.ts](../../server/src/index.ts), `scheduleGitHubConnectionEventPoll` and + `scheduleGitHubConnectionContinuitySweep`. + +- [ ] **JOB-15 — Merged pull-request confirmations.** Current: **30s** pending + confirmation scan, followed by GitHub checks for matching candidates. Change: + activate on merge events with bounded polling fallback only while candidates + exist. **Complete when:** confirmation resolution remains authorized and + restart-safe, with no empty scan. Source: + [issue-thread-interactions.ts](../../server/src/services/issue-thread-interactions.ts). + +- [ ] **JOB-16 — Terminal workspace cleanup.** Current: **30s** candidate scans. + Change: enqueue from terminal transitions and cooldown deadlines. **Complete + when:** empty state is quiet, and cleanup still checks current task state, + artifacts, deliveries, and ownership before deleting resources. Source: + [execution-workspaces.ts](../../server/src/services/execution-workspaces.ts). + +- [ ] **JOB-17 — Sandbox cleanup and orphan spools.** Current: **30s** scans even + when attempts have longer staleness/backoff. Change: durable cleanup queue with + next retry, immediate failed-release activation, and provider/fleet backstop. + **Complete when:** idle scans stop without leaking paid sandboxes or ignoring + malformed/unpersisted spool entries. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `sweepPendingCleanupLeases`. + +- [ ] **JOB-18 — Adapter login and setup-token reapers.** Current: **30s** + durable-session/lease scans, alongside local one-shot timers. Change: register + actual expiry and cleanup retry times. **Complete when:** abandoned logins and + interrupted teardown still release sandboxes across restart, without scans + when no obligations exist. Source: + [index.ts](../../server/src/index.ts), adapter and setup-token reaper wiring. + +- [ ] **JOB-19 — Custom-image setup expiry.** Current: **30s** active-session + expiry scans. Change: register `expiresAt` and cleanup attempts. **Complete + when:** no sessions means no SQL and expired provider resources are released. + Source: [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts). + +- [ ] **JOB-20 — Tool connection health.** Current: **30s** DB inventory; + provider probes are conditional. Change: check on use/configuration change, + with explicit deadlines for any promised proactive health checks. **Complete + when:** quiet connections stop creating empty scans and failures remain + visible within the agreed freshness contract. Source: + [index.ts](../../server/src/index.ts), `tools.sweepConnectionHealth`. + +- [ ] **JOB-21 — Secret proposals and expired action tokens.** Current: **30s** + expiry/cleanup queries. Change: enforce expiry synchronously on use and + register required scrub/cleanup deadlines. **Complete when:** idle cleanup + does not extend authorization or secret-retention periods. Sources: + [secret-proposals.ts](../../server/src/services/secret-proposals.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-22 — Chat completion delivery.** Current: **30s** outbox scan plus + an existing postcommit fast path. Change: extend that signal and register + retries only while pending. **Complete when:** committed completions survive + restart, duplicate activation, and provider failure without empty polling. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-23 — Connection continuation delivery.** Current: **30s** outbox + scans. Change: committed signals, retry deadlines, and startup reconciliation. + **Complete when:** newly available connections resume eligible work once, + preserve execution gates, and have no empty SQL loop. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-24 — Tool action reviews and deliveries.** Current: **30s** + review/session/outbox scans. Change: register review expiry and delivery + deadlines; activate after committed decisions. **Complete when:** approval, + rejection, expiry, and delivery retain authorization and replay protection. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-25 — Question response delivery.** Current: **30s** outbox scans. + Change: committed-answer activation, durable retry deadlines, and startup + recovery. **Complete when:** responses preserve attempt-generation fencing + and recover from delivery failure without polling empty queues. Source: + [question-response-delivery.ts](../../server/src/services/question-response-delivery.ts). + +- [ ] **JOB-26 — Cost accounting and decision-model recovery.** Current: + **30s** reconciliation even when empty. Change: pending-receipt signals, + spool awareness, and durable retries. **Complete when:** no empty queries, + but unresolved receipts/reservations remain tracked and cannot be discarded + to permit sleep. Source: [heartbeat.ts](../../server/src/services/heartbeat.ts), + `reconcileCostAccounting`. + +- [ ] **JOB-27 — Run recovery, scheduled retries, and queued work.** Current: + **30s** orphan/retry/queue/stranded-assignment scans. Change: remain responsive + while work exists, register retry/control deadlines, and reconcile on startup + or wake. **Complete when:** quiet state has no scans and interrupted runs do + not duplicate provider work or lose continuation. Source: + [index.ts](../../server/src/index.ts), periodic heartbeat recovery chain. + +- [ ] **JOB-28 — Dependencies, task watchdogs, silent runs, and stale locks.** + Current: additional **30s** scans. Change: dependency/run transition signals, + active-run-only monitoring, actual timeout deadlines, and startup backstops. + **Complete when:** recovery remains scoped and fenced, and a completed task + still receives any owed watchdog review. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts). + +- [ ] **JOB-29 — Automatic database backups.** Current: enabled by default, + typically **hourly**, with separate dump connections. Change: register with + the internal scheduler; later assign durable waking or explicitly transfer + backup responsibility to managed infrastructure. **Complete when:** backup + and restore guarantees are demonstrated, active backup blocks sleep, and a + merely configured future backup no longer blocks indefinitely once covered + by an external alarm. Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-30 — Replace the main tick and its disabled branch.** Current: + default **30s** main tick; setting `HEARTBEAT_SCHEDULER_ENABLED=false` still + leaves accounting, external-object, cleanup, and GitHub work, plus independent + workers. Change: migrate all lanes before retiring aggregate polling. + **Complete when:** disabled scheduling semantics remain documented and no + legacy fallback quietly reintroduces recurring SQL. Sources: + [index.ts](../../server/src/index.ts), [config.ts](../../server/src/config.ts). + +## 5. Connectors and transport lifetime + +- [ ] **NET-01 — AgentMail socket ownership.** Current: outbound WebSocket and + **20s** DB lease renewal; its renewal bypasses the top-level background gate. + Change: prefer webhook mode for hosted stacks; otherwise retain an awake + receiver or use shared durable ingress. **Complete when:** explicit suspend + closes all associated timers and message reception/replay still works. + Source: [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-02 — AgentMail catch-up.** Current: roughly **one-minute** provider + reconciliation even in webhook mode. Change: catch up on wake/connect and at + an owned recovery deadline. **Complete when:** checkpoints and deduplication + survive lost webhooks, with a defined maximum recovery delay. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-03 — Discord Gateway.** Current: persistent connection and **5s** DB + ownership renewal. Change: keep unsupported stacks awake initially; evaluate + a shared ingress worker owning the connection, lease, and durable delivery. + **Complete when:** idle tenant sleep does not silently stop timely reception + or weaken ownership fencing. Sources: + [chat-channels.ts](../../server/src/services/chat-channels.ts), + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-04 — Photon iMessage.** Current: several gRPC subscriptions, **5s** + ownership renewal, and **15s** catch-up. Change: explicit transport lifecycle + and, where required, shared durable receiving. **Complete when:** resume + preserves ordered checkpoints and detects replay/history gaps; stopping the + top-level chat loop also retires dependent work safely. Source: + [receiver.ts](../../server/src/services/photon/receiver.ts). + +- [ ] **NET-05 — Slack webhook ingress.** Current transport already uses webhook + mode. Change: qualify it through wake-aware ingress rather than introducing + a persistent receiver. **Complete when:** cold start meets provider timing + through durable acknowledgement/retry where necessary, with signature + verification, deduplication, and company scope preserved. Source: + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-06 — HTTP idle connections.** Current server keepalive is **185s** + to outlive common proxy defaults. Change: explicitly close idle sockets during + drain and qualify provider-specific connection policies. **Complete when:** + quiet sockets do not prevent intended sleep, and normal proxy reuse does not + gain intermittent 502/ECONNRESET failures. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **NET-07 — Active execution transports and lease renewals.** Inventory + runner/PRP connections, HTTP/2 bridge pings, native/legacy controller leases, + terminal SSH keepalives, browser viewport ownership, and per-operation claims. + Retain their required cadence during work. **Complete when:** active operations + hold the instance awake and unused transports retire; no arbitrary heartbeat + disabling is used to manufacture sleep. Sources: + [native-session-executor.ts](../../server/src/services/native-runtime/native-session-executor.ts), + [legacy-controller-lease.ts](../../server/src/services/legacy-controller-lease.ts), + [HTTP/2 bridge](../../packages/adapter-utils/src/http2-bridge-server.ts), + [terminal WebSocket](../../server/src/realtime/environment-custom-image-terminal-ws.ts). + +## 6. Browser demand + +These client-side schedules do not run in the server scheduler. Consolidate +their activity policy so a parked browser does not create artificial tenant +demand. Preserve real-time behavior while a person is actively using the app. + +- [ ] **UI-01 — Live WebSocket and fallback polling.** Hidden tabs already close + the main socket; visible tabs receive **30s** pings and use **15s** refetch + fallback when disconnected. Change: distinguish intentional sleep from + outage and decide whether unattended visible tabs may become inactive. + **Complete when:** reconnection does not create a self-wake loop and returning + users reconcile missed state. Source: + [LiveUpdatesProvider.tsx](../../ui/src/context/LiveUpdatesProvider.tsx). + +- [ ] **UI-02 — Stable page queries.** Current issue views poll at **1–3s**, + browser/artifact views at **3–5s**, and agents/costs/reviews/settings at + **5–60s**. Change: extend event invalidation and active-work-only fallback; + reuse [polling.ts](../../ui/src/lib/polling.ts) and + [useSharedPolling.ts](../../ui/src/hooks/useSharedPolling.ts). + **Complete when:** idle views stop artificial demand without hiding updates; + multiple tabs do not duplicate avoidable work. Starting sources: + [IssueDetail.tsx](../../ui/src/pages/IssueDetail.tsx), + [Costs.tsx](../../ui/src/pages/Costs.tsx), + [useTaskBrowsers.ts](../../ui/src/hooks/useTaskBrowsers.ts). + +- [ ] **UI-03 — Cloud-access error recovery.** Current transient-error retries + run every **5s**, explicitly in background tabs; bootstrap has faster polling. + Change: visibility/online gating and wake-aware recovery, with bounded + bootstrap behavior. **Complete when:** a hidden tab that last received 502/503 + cannot keep waking its tenant indefinitely. Source: + [CloudAccessGate.tsx](../../ui/src/components/CloudAccessGate.tsx). + +- [ ] **UI-04 — Service-worker update checks.** Current update check is + **hourly**, regardless of visibility. Change: check on foreground activity or + serve shell/update assets from shared static hosting. **Complete when:** a + parked hidden tab does not periodically wake its tenant to fetch `/sw.js`. + Source: [service-worker-updates.ts](../../ui/src/lib/service-worker-updates.ts). + +- [ ] **UI-05 — Plugin SSE streams.** Current EventSource lacks visibility + gating and holds an HTTP response open. Change: explicit visibility/idle + lifetime with reconciliation or replay on reconnect. **Complete when:** + plugin state remains correct and abandoned streams do not block sleep. + Sources: [bridge.ts](../../ui/src/plugins/bridge.ts), + [plugin routes](../../server/src/routes/plugins.ts). + +- [ ] **UI-06 — Setup and enrollment polling.** OAuth/chat setup often polls at + **1.5–3s**, email setup at **10s**, commonly while pending. Change: keep these + responsive during the bounded workflow; stop on terminal state, close, or + inactivity with a deliberate resume path. **Complete when:** abandoned setup + creates neither permanent demand nor lost completion evidence. Sources: + [ConnectionSetupFlow.tsx](../../ui/src/features/connections/ConnectionSetupFlow.tsx), + [chat setup](../../ui/src/pages/apps/chat/ChatEndpointSetup.tsx), + [email setup](../../ui/src/pages/apps/chat/EmailEndpointSetup.tsx). + +## 7. Lower traffic housekeeping and plugins + +- [ ] **AUX-01 — First-party telemetry.** The **60s** flush returns without + sending when its queue is empty. Change: register flush/retry work only when + needed and drain or durably retain pending batches. **Complete when:** empty + telemetry is network-quiet and delivery semantics remain explicit. Preserve + generated-contract, README, and privacy-review requirements for event changes. + Sources: [telemetry.ts](../../server/src/telemetry.ts), + [telemetry client](../../packages/shared/src/telemetry/client.ts). + +- [ ] **AUX-02 — Optional OpenTelemetry.** No configured OTLP endpoint means a + no-op tracer; this is separate from first-party telemetry and the local run + log. Change: inspect exporter batching/connection behavior only when enabled; + avoid a perpetual stream of idle-maintenance spans. **Complete when:** the + endpoint gate remains intact and configured exporters do not defeat intended + sleep after their pending work settles. Source: + [instrumentation.ts](../../server/src/instrumentation.ts). + +- [ ] **AUX-03 — Plugin log buffer.** The **5s** flush avoids SQL when empty. + Change: schedule a bounded flush after the first buffered entry and include + pending writes in drain. **Complete when:** no empty timer is needed and log + persistence has a defined shutdown/sleep boundary. Source: + [plugin-host-services.ts](../../server/src/services/plugin-host-services.ts). + +- [ ] **AUX-04 — Import spools and in-memory status expiry.** Hourly import + cleanup already avoids SQL when the local spool is absent/empty. Runtime + status expiry is memory-only. Change: use actual deadlines or opportunistic + cleanup where timing is not contractual. **Complete when:** neither causes an + unnecessary external wake, while retained files and required cleanup remain + tracked. Sources: + [company-import-transfers.ts](../../server/src/services/company-import-transfers.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **AUX-05 — Plugin worker lifecycle.** Enabled plugins can execute arbitrary + background work; host timers cannot prove they are quiet. Change: explicit + quiesce/resume, scheduled-job registration, ingress ownership, and lazy provider + worker startup where appropriate. **Complete when:** supported plugins prove + quiescence; unsupported plugins remain a named blocker. Source: + [plugin-worker-manager.ts](../../server/src/services/plugin-worker-manager.ts). + +- [ ] **AUX-06 — Verify low-priority and third-party sources at runtime.** + Announcements are request-driven with caching; no dedicated periodic server + application-update check was found. Plugin log retention has no production + caller in the audited revision. **Complete when:** packet traces confirm these + classifications and identify any SDK, gRPC, HTTP agent, monitoring, remote + client, or plugin traffic omitted by static inspection. Source: + [announcement-feed.ts](../../server/src/services/announcement-feed.ts). + +## 8. Sleep eligibility and alternative hosting + +- [ ] **SAFE-01 — Separate claimed inactivity from unclaimed warm standby.** + Existing warm standby bypasses SQL and recurring work only for empty, + unclaimed instances. Change: add claimed-instance behavior through explicit + scheduler/quiescence contracts. **Complete when:** an inactive claimed tenant + can become quiet without impersonating an unclaimed tenant or disrupting + signed claim handling. Source: + [cloud-warm-standby.ts](../../server/src/services/cloud-warm-standby.ts). + +- [ ] **SAFE-02 — Replace conservative blockers one capability at a time.** + Current safety blocks every enabled plugin, configured backups, API keys, + active routines/connections, nonterminal issues, and many retained records. + Change: distinguish current obligations from passive capability/history only + when a validated wake/recovery contract exists. **Complete when:** waiting for + a human or retaining completed history can permit sleep, while unknown state, + cleanup/accounting debt, and unhandled transports remain blockers. Sources: + [idle-sleep-safety.ts](../../server/src/services/idle-sleep-safety.ts), + [idle-local-work.ts](../../server/src/services/idle-local-work.ts). + +- [ ] **SAFE-03 — Validate cold start and stateful resume.** A snapshot preserves + memory, not the validity of external sockets, credentials, or leases. Change: + recheck deadlines and ownership on resume, recover stale connections, and + avoid replaying uncertain operations. **Complete when:** long sleeps, + expired sessions, and remote controller changes produce correct recovery + rather than duplicate execution or prolonged ingress rejection. + +- [ ] **HOST-01 — Evaluate colocated PostgreSQL separately.** Public authenticated + deployments currently reject an unset `DATABASE_URL` rather than falling back + to embedded PostgreSQL. Prototype explicit loopback PostgreSQL or deliberately + revise that deployment contract. **Complete when:** persistent storage, + backups/restores, upgrades, memory cost, and sleep/resume are qualified, with + agent execution isolated from the controller/database. This option does not + remove scheduler or wake requirements. Source: + [index.ts](../../server/src/index.ts), `assertCloudDatabaseContract`. + +## 9. Delivery order and acceptance + +1. Complete mechanism triage for the selected first slice: normal trigger, + producer coverage, durable dispatch, and remaining temporal obligations. +2. Ship DB-free health/probe handling and contained event-driven migrations. + Measure empty-worker SQL reduction while preserving delivery backstops until + their replacements are proven. These changes do not depend on a full scheduler. +3. Build the internal scheduler around true deadlines, starting with routines + and adding required retry/expiry/maintenance paths. Then add the external-waker + adapter and acknowledged alarm protocol before sleeping scheduled workloads. +4. Qualify connector transports, plugin lifecycle, browser demand, and provider + resume behavior. Expand eligibility only with evidence. + +- [ ] **TEST-01 — Measure an empty claimed tenant.** After startup settles, prove + zero app-originated SQL and no unintended outgoing packets, then observe actual + Neon and hosting sleep. Use provider/control-plane metrics so measurement + itself does not wake the tenant DB. Compare against the original baseline. + +- [ ] **TEST-02 — Prove first-use recovery.** After real sleep, perform an + authenticated read, write, and transaction. Verify persistent results, no + duplicate effects, and measured end-to-end latency including authentication + and database startup. + +- [ ] **TEST-03 — Prove scheduled work.** Exercise routines, a moved-earlier + deadline, cancellation, retry, expiry, and maintenance through restart/sleep. + Deliver duplicate/stale wakes and simulate external-waker failure. Verify + catch-up, claims, and awake fallback. + +- [ ] **TEST-04 — Prove event ingress.** Deliver a verified webhook while asleep + and while entering sleep. Exercise a connector with its supported receiver + strategy. Verify acknowledgements, replay, company scope, and persistence. + +- [ ] **TEST-05 — Prove active-work protection.** Attempt sleep during a run, + transaction, backup, finalization, cleanup attempt, and pending accounting + receipt. Verify the intended blocker/hold and safe release after settlement. + +- [ ] **TEST-06 — Prove browser inactivity behavior.** Test hidden tabs, + unattended visible tabs, reconnect/error states, plugin streams, and multiple + tabs. Verify returning users see reconciled state without an accidental wake + storm. + +- [ ] **TEST-07 — Prove crash, ownership, and rollout behavior.** Test a crash + around commit/alarm publication, stale drain owner, provider stop racing a + deployment, and long stateful suspension. Roll back migrated lanes without + duplicate workers. Measure fleet wake concurrency and actual compute savings + before broad enablement. + +Each implementation PR should run the smallest relevant checks first and the +repository-required verification for its handoff scope. This documentation-only +checklist does not claim any provider sleep/wake test has passed. diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index c1b34d21eb6..3f6061deaae 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -3,6 +3,7 @@ import { drizzle as drizzlePg } from "drizzle-orm/postgres-js"; import { readFile, readdir } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import postgres from "postgres"; +import { installDatabaseWorkSignals } from "./work-signals.js"; import * as schema from "./schema/index.js"; const MIGRATIONS_FOLDER = fileURLToPath(new URL("./migrations", import.meta.url)); @@ -268,6 +269,10 @@ export async function withDedicatedDbConnection(db: Db, action: (dedicated: D export function createDb(url: string, options?: DatabaseClientOptions) { const resolved = resolveDatabaseClientOptions(options ?? databaseClientOptionsFromEnv()); + return createDbClient(url, resolved); +} + +function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object) { const sql = postgres(url, postgresJsOptions(resolved)); const key = hostPortKeyOrNull(url); if (key) registerClient(key, sql); @@ -275,10 +280,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) { // postgres.js calls that error "write CONNECTION_CLOSED" too, so the // message cannot establish that replay is safe. Leave retries to callers // that know the complete operation is idempotent. - const db = drizzlePg(sql, { schema }); - dedicatedDbFactories.set(db, () => createDb(url, { + const db = installDatabaseWorkSignals(drizzlePg(sql, { schema }), owner); + dedicatedDbFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: 1, applicationName: "paperclip-workspace-finalization-lock", - })); + }, db)); return db; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 378c9bd37b8..73c6028d067 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -42,3 +42,5 @@ export { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecy export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; + +export { signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts new file mode 100644 index 00000000000..9bc3f738b1c --- /dev/null +++ b/packages/db/src/work-signals.test.ts @@ -0,0 +1,165 @@ +import { sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb, withDedicatedDbConnection } from "./client.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; +import { databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("database work signals", () => { + let db: ReturnType; + let temporary: Awaited>; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("work-signals-"); + db = createDb(temporary.connectionString); + await db.execute(sql`create table work_signal_test (id text primary key)`); + }, 90_000); + afterAll(async () => { await temporary?.cleanup(); }); + + it("defers nested savepoint notifications until the outer commit and preserves return values", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "nested", event => events.push(event)); + try { + const result = await db.transaction(async tx => { + await tx.transaction(async nested => { + await signalDatabaseWork(nested, "nested"); + await nested.execute(sql`insert into work_signal_test values ('nested')`); + }); + expect(events).toEqual(["intent"]); + expect(databaseWorkPending(db, "nested")).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(0); + return 123; + }); + expect(result).toBe(123); + expect(events).toEqual(["intent", "settled"]); + expect(databaseWorkPending(db, "nested")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(1); + } finally { unsubscribe(); } + }); + + it("reconciles a rollback without leaving a permanent pending flag", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "rollback"); + await tx.execute(sql`insert into work_signal_test values ('rollback')`); + throw new Error("abort"); + })).rejects.toThrow("abort"); + await reconcileDatabaseWork(db, "rollback"); + expect(databaseWorkPending(db, "rollback")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'rollback'`)).toHaveLength(0); + }); + + it("keeps outer ownership after a caught savepoint rollback", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "savepoint", event => events.push(event)); + try { + await db.transaction(async tx => { + await expect(tx.transaction(async nested => { + await signalDatabaseWork(nested, "savepoint"); + await nested.execute(sql`insert into work_signal_test values ('savepoint')`); + throw new Error("rollback savepoint"); + })).rejects.toThrow("rollback savepoint"); + expect(databaseWorkPending(db, "savepoint")).toBe(true); + expect(events).toEqual(["intent"]); + await signalDatabaseWork(tx, "savepoint"); + await tx.execute(sql`insert into work_signal_test values ('savepoint-retry')`); + }); + expect(events).toEqual(["intent", "intent", "settled"]); + expect(databaseWorkPending(db, "savepoint")).toBe(false); + } finally { unsubscribe(); } + }); + + it.each(["commit", "abort"])("does not clear a late %s until PostgreSQL reports a settled transaction", async outcome => { + // Inject a lost response at the client boundary while a REAL transaction + // remains open on PostgreSQL. The status probe and row visibility are real. + const raw = drizzle(db.$client); + const transaction = raw.transaction.bind(raw); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let wrote!: () => void; + const written = new Promise(resolve => { wrote = resolve; }); + let serverDone!: Promise; + raw.transaction = ((callback: any) => { + serverDone = transaction(async tx => { + const result = await callback(tx); + wrote(); + await gate; + if (outcome === "abort") throw new Error("server abort"); + return result; + }); + // Observe rejection immediately; tests await the same result below. + void serverDone.catch(() => {}); + return written.then(() => { throw new Error("lost transaction response"); }); + }) as typeof raw.transaction; + const owner = installDatabaseWorkSignals(raw); + const topic = `late-${outcome}`; + try { + await expect(owner.transaction(async tx => { + await signalDatabaseWork(tx, topic); + await tx.execute(sql`insert into work_signal_test values (${topic})`); + })).rejects.toThrow("lost transaction response"); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(0); + release(); await serverDone.catch(() => {}); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(outcome === "commit" ? 1 : 0); + } finally { release(); await serverDone?.catch(() => {}); } + }); + + it("recovers after a real backend disconnect without replaying the queue write", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "disconnect"); + await tx.execute(sql`insert into work_signal_test values ('disconnect')`); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + await db.execute(sql`select pg_terminate_backend(${pid as number})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + await reconcileDatabaseWork(db, "disconnect"); + expect(databaseWorkPending(db, "disconnect")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'disconnect'`)).toHaveLength(0); + }); + + it("shares notifications with dedicated child pools but isolates unrelated roots and topics", async () => { + const listener = vi.fn(); + const other = createDb(temporary.connectionString); + const unsubscribe = subscribeDatabaseWork(db, "dedicated", listener); + try { + await other.transaction(tx => signalDatabaseWork(tx, "dedicated")); + await db.transaction(tx => signalDatabaseWork(tx, "other")); + expect(listener).not.toHaveBeenCalled(); + await withDedicatedDbConnection(db, dedicated => dedicated.transaction(tx => signalDatabaseWork(tx, "dedicated"))); + expect(listener.mock.calls.map(([event]) => event)).toEqual(["intent", "settled"]); + } finally { unsubscribe(); await other.$client.end(); } + }); + + it("never changes transaction success because a listener throws", async () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const unsubscribe = subscribeDatabaseWork(db, "listener", () => { throw new Error("listener"); }); + try { + await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "listener"); return 7; })).resolves.toBe(7); + expect(warn).toHaveBeenCalledTimes(2); + } finally { unsubscribe(); warn.mockRestore(); } + }); +}); + +describe("work signal query boundaries", () => { + it("adds no SQL to transactions which do not register queue work", async () => { + const execute = vi.fn(); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + await expect(connection.transaction(async () => 19)).resolves.toBe(19); + expect(execute).not.toHaveBeenCalled(); + }); + it("fails before the write if XID acquisition fails, without retaining uncertainty", async () => { + const execute = vi.fn().mockRejectedValue(new Error("disconnected")); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + const write = vi.fn(); + await expect(connection.transaction(async (tx: any) => { await signalDatabaseWork(tx, "failed"); write(); })).rejects.toThrow("disconnected"); + expect(write).not.toHaveBeenCalled(); + expect(databaseWorkPending(connection, "failed")).toBe(false); + }); + it("rejects registration on an autocommit root", async () => { + const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() }); + await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction"); + }); +}); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts new file mode 100644 index 00000000000..219ea924468 --- /dev/null +++ b/packages/db/src/work-signals.ts @@ -0,0 +1,109 @@ +import { sql } from "drizzle-orm"; + +// Process-local hints about existing durable queues. No SQL triggers, tables, +// background timers, or driver query replay. Only opted-in transactions fetch +// an XID, so unrelated transactions keep their existing SQL behavior. +type Listener = (event: "intent" | "settled") => void; +type Transaction = { topics: Set; xid?: Promise; uncertain: boolean }; +type Scope = { listeners: Map>; transactions: Set; root: Connection }; +type Context = { scope: Scope; transaction?: Transaction }; +type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; +const contexts = new WeakMap(); + +function publish(scope: Scope, topic: string, event: "intent" | "settled" = "settled") { + for (const listener of scope.listeners.get(topic) ?? []) { + try { listener(event); } + catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } +} + +/** Installed once by createDb; dedicated child pools share their owner's scope. */ +export function installDatabaseWorkSignals(db: T, owner?: object): T { + const scope = (owner && contexts.get(owner)?.scope) || { listeners: new Map(), transactions: new Set(), root: db }; + install(db, { scope }); + return db; +} + +function install(connection: Connection, context: Context) { + contexts.set(connection, context); + const transaction = connection.transaction; + connection.transaction = async function (callback, ...options) { + // Nested savepoints belong to the outermost transaction. Even a released + // savepoint must not wake a consumer before its parent commits. + const state = context.transaction ?? { topics: new Set(), uncertain: false }; + try { + const result = await transaction.call(this, async (tx: Connection) => { + install(tx, { scope: context.scope, transaction: state }); + return callback(tx); + }, ...options); + if (!context.transaction) context.scope.transactions.delete(state); + return result; + } catch (error) { + if (!context.transaction) { + // An XID is known before the first opted-in write can execute. If + // acquisition itself failed there cannot be a corresponding write. + if (state.xid) state.uncertain = true; + else context.scope.transactions.delete(state); + } + throw error; + } finally { + if (!context.transaction) for (const topic of state.topics) publish(context.scope, topic); + } + }; +} + +/** Await BEFORE the queue write, on its transaction. Nested transactions work. */ +export async function signalDatabaseWork(connection: object, topic: string): Promise { + const context = contexts.get(connection); + if (!context) return; // Lightweight service test doubles have no DB lifecycle. + const state = context.transaction; + if (!state) throw new Error("Database work must be registered inside a transaction"); + state.topics.add(topic); + context.scope.transactions.add(state); + publish(context.scope, topic, "intent"); + if (!state.xid) { + state.xid = (connection as Connection).execute(sql`select pg_current_xact_id()::text as xid`) + .then((rows: { xid: string }[]) => rows[0]!.xid); + } + // Keep a failed acquisition distinguishable from an uncertain queue write. + try { await state.xid; } + catch (error) { state.xid = undefined; throw error; } +} + +export function databaseWorkPending(owner: object, topic: string): boolean { + return [...(contexts.get(owner)?.scope.transactions ?? [])].some(state => state.topics.has(topic)); +} + +/** Resolve only failed transactions, on a fresh root-pool statement. Call + * BEFORE the queue scan so rows from a newly confirmed commit are visible. + * NULL means the XID aged out; it can no longer be in progress, so scanning + * the durable queue is sufficient even though its outcome is unavailable. + */ +export async function reconcileDatabaseWork(owner: object, topic: string): Promise { + const scope = contexts.get(owner)?.scope; + if (!scope) return; + for (const state of [...scope.transactions]) { + if (!state.uncertain || !state.topics.has(topic) || !state.xid) continue; + const xid = await state.xid; + const rows = await scope.root.execute(sql`select pg_xact_status(${xid}::xid8) as status`); + if (rows[0]?.status === "in progress") continue; + if (!rows.length || ![null, "committed", "aborted"].includes(rows[0].status)) { + throw new Error("Unexpected database transaction status"); + } + scope.transactions.delete(state); + for (const affected of state.topics) publish(scope, affected); + } +} + +/** Includes intent, outer settlement, and uncertainty-resolution changes. */ +export function subscribeDatabaseWork(owner: object, topic: string, listener: Listener): () => void { + const context = contexts.get(owner); + if (!context) throw new Error("Database work subscriptions require createDb"); + let listeners = context.scope.listeners.get(topic); + if (!listeners) context.scope.listeners.set(topic, listeners = new Set()); + listeners.add(listener); + if (databaseWorkPending(owner, topic)) { + try { listener("intent"); } catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } + return () => { listeners.delete(listener); }; +} diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index bb8e7f076db..694f511d0e5 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -1,3 +1,5 @@ +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { subscribeDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,7 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; -import { issueService } from "../services/issues.js"; +import { executeIssuePostCommitActions, type IssuePostCommitAction, issueService } from "../services/issues.js"; import { buildLowTrustSourceTrust } from "../services/source-trust.js"; import { documentService } from "../services/documents.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -51,6 +53,57 @@ const support = await getEmbeddedPostgresTestSupport(); }; return { companyId, agentId, sourceId, runId, task, create, finish, rows, due, wakeup, service, run }; } + it("dispatches a committed completion without activity publication and recovers after restart", async () => { + const f = await seed(); + const run = vi.fn(() => f.service.sweepPending()); + const makeWorker = () => { + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const worker = coordinator.register(DELIVERY_QUEUES.chatCompletion, { retryMs: 60_000, run, hasPending: f.service.hasPending }); + return { ...worker, stop: coordinator.stop }; + }; + const worker = makeWorker(); + try { + await worker.ready; + expect(await f.service.hasPending()).toBe(false); + expect(run).toHaveBeenCalledTimes(1); + await f.finish(); // Service mutation alone: no route activity callback. + await vi.waitFor(() => expect(f.wakeup).toHaveBeenCalledTimes(1)); + expect(await f.service.hasPending()).toBe(true); // queued receipt still owed + } finally { await worker.stop(); } + // Admit the first reporting turn; later completions need a new turn. + await f.run(); + // A second task commits while no consumer is registered. + const next = await f.create(); + await f.finish(next.id); + const restarted = makeWorker(); + try { + await restarted.ready; + expect(f.wakeup).toHaveBeenCalledTimes(2); + } finally { await restarted.stop(); } + }); + it("automatically notifies after a caller-owned transaction commits", async () => { + const f = await seed(); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatCompletion, notified); + try { + const actions: IssuePostCommitAction[] = []; + await db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "done" }, tx, [], actions); + expect(notified).not.toHaveBeenCalled(); + }); + expect(notified).toHaveBeenCalledTimes(1); + expect(actions).toEqual([]); + await executeIssuePostCommitActions(db, actions); + expect(notified).toHaveBeenCalledTimes(1); + const rolledBack: IssuePostCommitAction[] = []; + await expect(db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "todo" }, tx, [], rolledBack); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + expect(notified).toHaveBeenCalledTimes(1); + expect((await f.rows())[0]?.status).toBe("pending"); + } finally { unsubscribe(); } + }); it("records authenticated origins and atomically creates one event per Done transition", async () => { const f = await seed(); expect(await db.select().from(handoffs).where(eq(handoffs.taskId, f.task.id))).toMatchObject([{ conversationId: f.sourceId, sessionGeneration: 0 }]); diff --git a/server/src/__tests__/connection-intents-service.test.ts b/server/src/__tests__/connection-intents-service.test.ts index c6c318c372d..94613ec1f33 100644 --- a/server/src/__tests__/connection-intents-service.test.ts +++ b/server/src/__tests__/connection-intents-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; @@ -773,7 +775,11 @@ describeEmbeddedPostgres("connectionIntentService", () => { it("recovers a resolution after a failed dispatch and acknowledges an already queued wake exactly once", async () => { const service = connectionIntentService(db); const pending = await service.request(claims, "airtable"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.connection, notified); await service.decline(pending.interactionId!, claims.responsible_user_id); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); const wakeup = vi.fn().mockRejectedValueOnce(new Error("simulated crash before durable enqueue")); const deliveries = connectionIntentDeliveryService(db, { wakeup } as never); await expect(deliveries.deliver(pending.interactionId!)).rejects.toThrow("simulated crash"); diff --git a/server/src/__tests__/delivery-work-coordinator.test.ts b/server/src/__tests__/delivery-work-coordinator.test.ts new file mode 100644 index 00000000000..1be5395f9df --- /dev/null +++ b/server/src/__tests__/delivery-work-coordinator.test.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { for (const c of coordinators.splice(0)) await c.stop(); vi.useRealTimers(); }); +function setup() { + vi.useFakeTimers(); + let outcome = "committed"; + let rejectCommit = false; + const execute = vi.fn(async () => [{ xid: "42", status: outcome }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + const result = await callback({ execute, transaction: owner.transaction }); + if (rejectCommit) throw new Error("lost commit reply"); + return result; + } }); + const canRun = vi.fn(() => true), canReconcile = vi.fn(() => false), onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile, onError }); + coordinators.push(coordinator); + const enqueue = (queue = DELIVERY_QUEUES.feedback as typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]) => + owner.transaction(tx => notifyDeliveryWork(tx, queue)); + return { owner, canRun, canReconcile, onError, coordinator, enqueue, execute, + failCommit: () => { rejectCommit = true; outcome = "in progress"; }, settle: (status = "committed") => { outcome = status; } }; +} +function task() { return { retryMs: 5000, run: vi.fn(async () => {}), hasPending: vi.fn(async () => false) }; } + +describe("delivery work coordinator", () => { + it("leaves five empty queues with no deadlines, timers, or periodic queries", async () => { + const s = setup(); + const tasks = Object.values(DELIVERY_QUEUES).map(queue => { + const t = task(); return { queue, t, worker: s.coordinator.register(queue, t) }; + }); + await Promise.all(tasks.map(t => t.worker.ready)); + expect(vi.getTimerCount()).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(3_600_000); + for (const { queue, t } of tasks) { + expect(t.run).toHaveBeenCalledTimes(1); + await s.enqueue(queue); await s.enqueue(queue); + } + await vi.advanceTimersByTimeAsync(1); + for (const { t } of tasks) expect(t.run).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not postpone an already committed wake when another producer starts writing", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + await s.enqueue(); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const writing = s.owner.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); + await gate; + }); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + release(); await writing; + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(3); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("fences unresolved writes until the server settles, then scans and releases the idle hold", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); + await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + expect(idleWorkSnapshot().active).toBe(1); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); // Empty scan cannot clear unresolved write. + expect(idleWorkSnapshot().active).toBe(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.settle(); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + it("retains unresolved intent across coordinator replacement", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); await expect(s.enqueue()).rejects.toThrow(); + await s.coordinator.stop(); + expect(idleWorkSnapshot().active).toBe(0); + const replacement = createDeliveryWorkCoordinator({ owner: s.owner, canRun: () => true, onError: vi.fn() }); + coordinators.push(replacement); + await replacement.register(DELIVERY_QUEUES.feedback, t).ready; + expect(idleWorkSnapshot().active).toBe(1); + s.settle(); await vi.advanceTimersByTimeAsync(5001); + expect(idleWorkSnapshot().active).toBe(0); + expect(replacement.nextWakeAt()).toBeNull(); + }); + it("retries pending deliveries and transient failures, then goes quiet", async () => { + const s = setup(), t = task(); + t.run.mockRejectedValueOnce(new Error("database disconnected")); + t.hasPending.mockResolvedValueOnce(true); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + expect(s.onError).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(10_000); + expect(t.run).toHaveBeenCalledTimes(3); + expect(s.coordinator.nextWakeAt()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not overlap a sweep or lose a commit during its final pending check", async () => { + const s = setup(), t = task(); + let release!: (pending: boolean) => void; + t.hasPending.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.connection, t); + await vi.advanceTimersByTimeAsync(1); + await s.enqueue(DELIVERY_QUEUES.connection); + await vi.advanceTimersByTimeAsync(1000); + expect(t.run).toHaveBeenCalledTimes(1); + release(false); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + }); + it("reconciles a rollback during idle drain and releases the empty-queue hold without dispatching", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + s.failCommit(); await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + await vi.advanceTimersByTimeAsync(1); + expect(idleWorkSnapshot().active).toBe(1); + expect(t.run).toHaveBeenCalledTimes(1); + s.settle("aborted"); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(t.hasPending).toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps committed pending work recoverable during drain without starting deliveries", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + t.hasPending.mockResolvedValue(true); + await s.enqueue(); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.canRun.mockReturnValue(true); + t.hasPending.mockResolvedValue(false); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(2); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not query during standby and resumes when admission opens", async () => { + const s = setup(), t = task(); s.canRun.mockReturnValue(false); + await s.coordinator.register(DELIVERY_QUEUES.toolAction, t).ready; + await s.enqueue(DELIVERY_QUEUES.toolAction); + await vi.advanceTimersByTimeAsync(120_000); + expect(t.run).not.toHaveBeenCalled(); + expect(t.hasPending).not.toHaveBeenCalled(); + s.canRun.mockReturnValue(true); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("awaits in-flight work on shutdown and releases the subscription for restart", async () => { + const s = setup(), t = task(); + let release!: () => void; + t.run.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.chatCompletion, t); + await vi.advanceTimersByTimeAsync(1); + const stopped = vi.fn(); + const stopping = s.coordinator.stop().then(stopped); + await s.enqueue(DELIVERY_QUEUES.chatCompletion); + expect(stopped).not.toHaveBeenCalled(); + release(); await stopping; + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("cancels an abortable delivery before awaiting shutdown", async () => { + const s = setup(); + const hasPending = vi.fn(async () => false); + const run = vi.fn((signal: AbortSignal) => new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + })); + s.coordinator.register(DELIVERY_QUEUES.feedback, { retryMs: 5000, run, hasPending }); + await vi.advanceTimersByTimeAsync(1); + await s.coordinator.stop(); + expect(run.mock.calls[0]?.[0].aborted).toBe(true); + expect(hasPending).not.toHaveBeenCalled(); + expect(s.onError).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/server/src/__tests__/feedback-service.test.ts b/server/src/__tests__/feedback-service.test.ts index ed3c7f21898..08a44442038 100644 --- a/server/src/__tests__/feedback-service.test.ts +++ b/server/src/__tests__/feedback-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -800,6 +802,8 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { sessionId, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.feedback, notified); await flushingSvc.saveIssueVote({ issueId, targetType: "issue_comment", @@ -808,7 +812,12 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { authorUserId: "user-1", allowSharing: true, }); - await flushingSvc.flushPendingFeedbackTraces(); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + // The route and worker may request a flush together after the commit. + await Promise.all([flushingSvc.flushPendingFeedbackTraces(), flushingSvc.flushPendingFeedbackTraces()]); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); expect(uploadTraceBundle).toHaveBeenCalledTimes(1); const bundle = uploadTraceBundle.mock.calls[0]?.[0] as Record | undefined; @@ -1152,6 +1161,30 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { expect(uploadTraceBundle).toHaveBeenCalledTimes(1); }); + it("leaves aborted and unstarted exports available for recovery after shutdown", async () => { + const first = await seedIssueWithAgentComment(); + const second = await seedIssueWithAgentComment(); + const shutdown = new AbortController(); + const uploadTraceBundle = vi.fn().mockResolvedValue({ objectKey: "saved" }) + .mockImplementationOnce(async (_bundle, signal: AbortSignal) => { + shutdown.abort(new Error("shutdown")); + signal.throwIfAborted(); + }); + const flushingSvc = feedbackService(db, { shareClient: { uploadTraceBundle } }); + for (const item of [first, second]) { + await flushingSvc.saveIssueVote({ issueId: item.issueId, targetType: "issue_comment", + targetId: item.commentId, vote: "up", authorUserId: "user-1", allowSharing: true }); + } + expect(await flushingSvc.flushPendingFeedbackTraces({ signal: shutdown.signal })) + .toMatchObject({ attempted: 1, sent: 0, failed: 1 }); + const rows = await db.select().from(feedbackExports); + expect(rows.filter(row => row.status === "failed")).toHaveLength(1); + expect(rows.filter(row => row.status === "pending")).toHaveLength(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + expect(await flushingSvc.flushPendingFeedbackTraces()).toMatchObject({ attempted: 2, sent: 2, failed: 0 }); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); + }); + it("marks pending shared traces as failed when no feedback export backend is configured", async () => { const { companyId, issueId, commentId } = await seedIssueWithAgentComment(); diff --git a/server/src/__tests__/feedback-share-client.test.ts b/server/src/__tests__/feedback-share-client.test.ts index fe710262839..d47d01ede4e 100644 --- a/server/src/__tests__/feedback-share-client.test.ts +++ b/server/src/__tests__/feedback-share-client.test.ts @@ -1,6 +1,6 @@ import { gunzipSync } from "node:zlib"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; +import { FEEDBACK_UPLOAD_TIMEOUT_MS, createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; describe("feedback trace share client", () => { beforeEach(() => { @@ -12,6 +12,7 @@ describe("feedback trace share client", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); it("defaults to telemetry.paperclip.ing when no backend url is configured", async () => { @@ -98,4 +99,43 @@ describe("feedback trace share client", () => { expect(parsed.objectKey.endsWith("/export-1.json")).toBe(true); expect(parsed.bundle.envelope).toEqual({ hello: "world" }); }); + it.each(["headers", "body"])("cancels a stalled upload at its deadline while waiting for %s", async phase => { + const deadline = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation(async (_url, options) => { + const block = () => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + }); + if (phase === "headers") return block(); + return { ok: true, json: block } as unknown as Response; + }); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const uploading = client.uploadTraceBundle(bundle); + const rejected = expect(uploading).rejects.toThrow("upload deadline"); + await waiting; + expect(timeout).toHaveBeenCalledWith(FEEDBACK_UPLOAD_TIMEOUT_MS); + deadline.abort(new Error("upload deadline")); + await rejected; + }); + + it("passes shutdown cancellation through to a pending upload", async () => { + const shutdown = new AbortController(); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation((_url, options) => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + })); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const rejected = expect(client.uploadTraceBundle(bundle, shutdown.signal)).rejects.toThrow("shutdown"); + await waiting; + shutdown.abort(new Error("shutdown")); + await rejected; + }); + }); diff --git a/server/src/__tests__/issue-feedback-routes.test.ts b/server/src/__tests__/issue-feedback-routes.test.ts index e59fe6d1e91..643956ae223 100644 --- a/server/src/__tests__/issue-feedback-routes.test.ts +++ b/server/src/__tests__/issue-feedback-routes.test.ts @@ -8,6 +8,7 @@ const mockFeedbackService = vi.hoisted(() => ({ listIssueVotesForUser: vi.fn(), listFeedbackTraces: vi.fn(), saveIssueVote: vi.fn(), + flushPendingFeedbackTraces: vi.fn(() => new Promise(() => {})), })); const mockIssueService = vi.hoisted(() => ({ @@ -18,9 +19,6 @@ const mockIssueService = vi.hoisted(() => ({ findMentionedAgents: vi.fn(), })); -const mockFeedbackExportService = vi.hoisted(() => ({ - flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 1, sent: 1, failed: 0 })), -})); const mockAccessService = vi.hoisted(() => ({ canUser: vi.fn(), hasPermission: vi.fn(), @@ -138,7 +136,7 @@ async function createApp(actor: Record) { (req as any).actor = actor; next(); }); - app.use("/api", issueRoutes({} as any, {} as any, { feedbackExportService: mockFeedbackExportService })); + app.use("/api", issueRoutes({} as any, {} as any)); const routeErrors: string[] = []; app.locals.routeErrors = routeErrors; app.use((error: unknown, _req: express.Request, _res: express.Response, next: express.NextFunction) => { @@ -152,11 +150,6 @@ async function createApp(actor: Record) { describe("issue feedback trace routes", () => { beforeEach(() => { vi.clearAllMocks(); - mockFeedbackExportService.flushPendingFeedbackTraces.mockResolvedValue({ - attempted: 1, - sent: 1, - failed: 0, - }); mockHeartbeatService.wakeup.mockResolvedValue(undefined); mockHeartbeatService.reportRunActivity.mockResolvedValue(undefined); mockHeartbeatService.getRun.mockResolvedValue(null); @@ -174,7 +167,7 @@ describe("issue feedback trace routes", () => { mockLogActivity.mockResolvedValue(undefined); }); - it("flushes a newly shared feedback trace immediately after saving the vote", async () => { + it("returns the saved vote without waiting for feedback uploads", async () => { const targetId = "11111111-1111-4111-8111-111111111111"; mockIssueService.getById.mockResolvedValue({ id: "issue-1", @@ -211,11 +204,9 @@ describe("issue feedback trace routes", () => { }); expect([200, 201]).toContain(res.status); - expect(mockFeedbackExportService.flushPendingFeedbackTraces).toHaveBeenCalledWith({ - companyId: "company-1", - traceId: "trace-1", - limit: 1, - }); + expect(mockFeedbackService.saveIssueVote).toHaveBeenCalledTimes(1); + expect(mockFeedbackService.flushPendingFeedbackTraces).not.toHaveBeenCalled(); + }); it("rejects non-board callers before fetching a feedback trace", async () => { diff --git a/server/src/__tests__/issue-thread-interactions-service.test.ts b/server/src/__tests__/issue-thread-interactions-service.test.ts index 6e909fc8b3a..887b0ea711d 100644 --- a/server/src/__tests__/issue-thread-interactions-service.test.ts +++ b/server/src/__tests__/issue-thread-interactions-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -231,11 +233,15 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => { expect(created.payload.questions.map((question) => question.id)).toEqual(["repo", "scope", "hosting"]); expect(await interactionsSvc.create(issue, input, { userId: "local-board" })).toEqual(created); await expect(interactionsSvc.answerQuestions(issue, created.id, { answers: [{ questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }] }, { userId: "local-board" })).rejects.toThrow("requires an answer"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.question, notified); const answered = await interactionsSvc.answerQuestions(issue, created.id, { answers: [ { questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }, { questionId: "scope", optionIds: ["all"] }, { questionId: "hosting", optionIds: ["existing", "new"] }, ] }, { userId: "local-board" }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); expect(answered.status).toBe("answered"); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 449d8d5b5dd..5eaa40e414d 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -44,11 +44,17 @@ const { const completionSweepMock = vi.fn(async () => undefined); const createAppMock = vi.fn(async () => Object.assign((_: unknown, __: unknown) => {}, { locals: { + deliveryWork: { + register: (_queue: string, task: { run: () => Promise }) => ({ + ready: task.run().catch(() => undefined), wake: vi.fn(), + }), + stop: vi.fn(async () => {}), + }, toolGateway: { sweepActionReviews: vi.fn(async () => ({ scanned: 0 })), cleanupExpiredSessions: vi.fn(async () => ({ deletedCount: 0 })), }, - toolActionDeliveries: { sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, + toolActionDeliveries: { hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, }, }) as never); const createBetterAuthInstanceMock = vi.fn(() => ({})); @@ -133,6 +139,7 @@ const { }; const routineServiceFactoryMock = vi.fn(() => routineServiceMock); const feedbackExportServiceMock = { + hasPendingFeedbackTraces: vi.fn(async () => false), flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 0, sent: 0, failed: 0 })), }; const feedbackServiceFactoryMock = vi.fn(() => feedbackExportServiceMock); @@ -363,16 +370,18 @@ vi.mock("../services/index.js", () => ({ })), })); -vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ sweepPending: completionSweepMock }) })); +vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ hasPending: vi.fn(async () => false), sweepPending: completionSweepMock }) })); vi.mock("../services/connection-intent-delivery.js", () => ({ connectionIntentDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, failed: 0 })), })), })); vi.mock("../services/question-response-delivery.js", () => ({ questionResponseDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, steered: 0, @@ -675,6 +684,22 @@ describe("startServer feedback export wiring", () => { expect(completionSweepMock).toHaveBeenCalled(); }); + it("keeps activity-driven completion scans scoped to the changed task", async () => { + await startServer(); + completionSweepMock.mockClear(); + const { publishLiveEvent } = await import("../services/live-events.js"); + publishLiveEvent({ companyId: "company-one", type: "activity.logged", payload: { + action: "issue.updated", entityId: "task-one", + } }); + expect(completionSweepMock).toHaveBeenCalled(); + for (const args of completionSweepMock.mock.calls) { + expect(args).toEqual([{ companyId: "company-one", taskId: "task-one" }]); + } + completionSweepMock.mockClear(); + publishLiveEvent({ companyId: "company-two", type: "activity.logged", payload: { action: "issue.updated" } }); + expect(completionSweepMock).not.toHaveBeenCalled(); + }); + it("never invokes the retired review detector at startup or on periodic recovery", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/__tests__/tool-gateway-service.test.ts b/server/src/__tests__/tool-gateway-service.test.ts index 3e9b984672d..0db14e6f47f 100644 --- a/server/src/__tests__/tool-gateway-service.test.ts +++ b/server/src/__tests__/tool-gateway-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -219,6 +221,8 @@ describeEmbeddedPostgres("tool gateway service", () => { runId: run.id, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.toolAction, notified); await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", @@ -228,6 +232,9 @@ describeEmbeddedPostgres("tool gateway service", () => { details: { instructions: expect.stringContaining("A human approval card was posted on task") }, }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", diff --git a/server/src/__tests__/work-scheduler.test.ts b/server/src/__tests__/work-scheduler.test.ts new file mode 100644 index 00000000000..dcfde0220a0 --- /dev/null +++ b/server/src/__tests__/work-scheduler.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createWorkScheduler } from "../services/work-scheduler.js"; +afterEach(() => vi.useRealTimers()); +describe("work scheduler", () => { + it("owns one timer, replaces named deadlines, and exposes the earliest wake", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(); + const first = vi.fn(), second = vi.fn(); + s.schedule("first", 100, first); + s.schedule("second", 50, second); + s.schedule("first", 20, first); + expect(vi.getTimerCount()).toBe(1); + expect(s.nextWakeAt()).toBe(20); + await vi.advanceTimersByTimeAsync(20); + expect(first).toHaveBeenCalledTimes(1); + expect(second).not.toHaveBeenCalled(); + expect(s.nextWakeAt()).toBe(50); + s.cancel("second"); + expect(s.nextWakeAt()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it("honors cancellation during dispatch and isolates a failing callback", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const s = createWorkScheduler(), cancelled = vi.fn(), later = vi.fn(); + s.schedule("first", 10, () => { s.cancel("cancelled"); throw new Error("failure"); }); + s.schedule("cancelled", 10, cancelled); + s.schedule("later", 20, later); + await vi.advanceTimersByTimeAsync(20); + expect(cancelled).not.toHaveBeenCalled(); + expect(later).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); s.stop(); + }); + it("does not fire distant deadlines early and cannot restart after stop", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(), run = vi.fn(); + s.schedule("far", 3_000_000_000, run); + await vi.advanceTimersByTimeAsync(2_147_483_647); + expect(run).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(852_516_353); + expect(run).toHaveBeenCalledTimes(1); + s.stop(); s.schedule("late", Date.now(), run); + expect(vi.getTimerCount()).toBe(0); + expect(() => createWorkScheduler().schedule("bad", NaN, run)).toThrow("finite"); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index bda8217716a..1697f311303 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,3 +1,5 @@ +import { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; import { isIdleTaskDrainActive, trackIdleWork } from "./services/task-admission.js"; import { customerSuccessRoutes } from "./routes/customer-success.js"; @@ -483,11 +485,13 @@ export async function createApp( serverPort: number; storageService: StorageService; feedbackExportService?: { + hasPendingFeedbackTraces(): Promise; flushPendingFeedbackTraces(input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; + signal?: AbortSignal; }): Promise; }; databaseBackupService?: InstanceDatabaseBackupService; @@ -877,7 +881,6 @@ export async function createApp( // route prefixes, so this dependency does not change issue-route precedence. api.use(issueRoutes(db, opts.storageService, { chatRunRetries: chatChannels, - feedbackExportService: opts.feedbackExportService, pluginWorkerManager: workerManager, approveToolActionRequest: (input) => toolGateway.approveActionRequest(input), declineToolActionRequest: (input) => toolGateway.declineActionRequest(input), @@ -1190,40 +1193,19 @@ export async function createApp( jobCoordinator.start(); scheduler.start(); - let feedbackExportShuttingDown = false; - let feedbackExportTimer: ReturnType | null = null; - const disableFeedbackExportFlushes = () => { - feedbackExportShuttingDown = true; - if (feedbackExportTimer) { - clearInterval(feedbackExportTimer); - feedbackExportTimer = null; - } - }; - const flushPendingFeedbackExports = async () => { - if (feedbackExportShuttingDown || isWarmStandby() || isIdleTaskDrainActive()) return; - try { - await opts.feedbackExportService?.flushPendingFeedbackTraces(); - } catch (err) { - if (isDatabaseConnectionUnavailableError(err)) { - disableFeedbackExportFlushes(); - logger.warn( - { err }, - "Disabling pending feedback export flushes because the database is unavailable", - ); - return; - } - logger.error({ err }, "Failed to flush pending feedback exports"); - } - }; - - feedbackExportTimer = opts.feedbackExportService - ? setInterval(() => { - void flushPendingFeedbackExports(); - }, FEEDBACK_EXPORT_FLUSH_INTERVAL_MS) - : null; - feedbackExportTimer?.unref?.(); + const deliveryWork = createDeliveryWorkCoordinator({ + owner: db, + canRun: () => !isWarmStandby() && !isIdleTaskDrainActive(), + canReconcile: () => !isWarmStandby(), + onError: (err, queue) => logger.error({ err, queue }, "Delivery reconciliation failed"), + }); + app.locals.deliveryWork = deliveryWork; if (opts.feedbackExportService) { - void flushPendingFeedbackExports(); + deliveryWork.register(DELIVERY_QUEUES.feedback, { + retryMs: FEEDBACK_EXPORT_FLUSH_INTERVAL_MS, + run: (signal) => opts.feedbackExportService!.flushPendingFeedbackTraces({ signal }), + hasPending: () => opts.feedbackExportService!.hasPendingFeedbackTraces(), + }); } emailChannels.start(); const flushChatPublications = async () => { @@ -1389,7 +1371,7 @@ export async function createApp( await publicMcpEvents?.stop(); await dotMcpEvents?.stop(); jobCoordinator.stop(); - disableFeedbackExportFlushes(); + await deliveryWork.stop(); unsubscribeChatPublicationSignals(); chatReconciliation.stop(); if (chatPublicationTimer) { diff --git a/server/src/index.ts b/server/src/index.ts index 4a887104178..6bef07ad23a 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,9 +1,11 @@ +import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; import { isIdleTaskDrainActive, beginIdleTrackedWork, trackIdleWork } from "./services/task-admission.js"; import { markIdleStartupComplete } from "./services/idle-local-work.js"; import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "./services/idle-database-backup.js"; import { cloudWarmStandbyServerOptions } from "./middleware/cloud-warm-standby.js"; import { createCloudWarmStandby } from "./services/cloud-warm-standby.js"; -import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; import { chatCompletionDeliveryService } from "./services/chat-completion-delivery.js"; /// // Kicks off the OTel bootstrap as early as possible (no-op unless @@ -1252,20 +1254,31 @@ async function startServerWithDatabaseTeardown( const ENVIRONMENT_LEASE_CLEANUP_SWEEP_BACKOFF_MS = 5 * 60 * 1000; const environmentLeaseCleanupHeartbeat = accountingHeartbeat; const chatCompletionDeliveries = chatCompletionDeliveryService(db as any, environmentLeaseCleanupHeartbeat); - // Activity publication happens after the status transaction commits. This is - // a best-effort fast path; the durable outbox and sweeps remain authoritative. - const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped || event.type !== "activity.logged" || - event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) - .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); - }); - server.on("close", unsubscribeChatCompletions); const connectionDeliveries = connectionIntentDeliveryService(db as any, environmentLeaseCleanupHeartbeat); const questionResponseDeliveries = questionResponseDeliveryService(db as any, { heartbeat: environmentLeaseCleanupHeartbeat, resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); + const deliveryWork = app.locals.deliveryWork as ReturnType; + const deliveryWorkers = ([ + [DELIVERY_QUEUES.chatCompletion, chatCompletionDeliveries], + [DELIVERY_QUEUES.connection, connectionDeliveries], + [DELIVERY_QUEUES.question, questionResponseDeliveries], + [DELIVERY_QUEUES.toolAction, app.locals.toolActionDeliveries as ReturnType], + ] as const).map(([queue, service]) => deliveryWork.register(queue, { + retryMs: config.heartbeatSchedulerIntervalMs, + run: () => service.sweepPending(), + hasPending: () => service.hasPending(), + })); + // Preserve the existing task-scoped activity fast path. Claims in the + // delivery service arbitrate with the coordinator's startup/recovery sweep. + const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { + if (isIdleTaskDrainActive() || isWarmStandby() || heartbeatSchedulerStopped || event.type !== "activity.logged" || + event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; + trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) + .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); + }); + server.on("close", unsubscribeChatCompletions); const runEnvironmentLeaseCleanupSweep = (backoffMs: number) => environmentLeaseCleanupHeartbeat .sweepPendingCleanupLeases({ backoffMs }) @@ -1316,18 +1329,9 @@ async function startServerWithDatabaseTeardown( })); }; - await chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "startup chat completion delivery recovery failed")); - await connectionDeliveries.sweepPending(); await app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "startup tool review recovery failed")); await app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); - await app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "startup tool review delivery sweep failed")); - await questionResponseDeliveries.sweepPending().then((result) => { - if (result.scanned > 0) { - logger.info(result, "startup question-response delivery sweep completed"); - } - }).catch((err) => { - logger.error({ err }, "startup question-response delivery sweep failed"); - }); + await Promise.all(deliveryWorkers.map(worker => worker.ready)); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); @@ -1781,23 +1785,10 @@ async function startServerWithDatabaseTeardown( logger.error({ err }, "periodic secret proposal expiry sweep failed"); })); - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "chat completion delivery failed"))); - trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed"))); - trackHeartbeatSchedulerWork(connectionDeliveries.sweepPending().catch((err) => logger.error({ err }, "connection continuation delivery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "tool review recovery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "gateway token cleanup failed"))); - trackHeartbeatSchedulerWork(app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "tool review delivery sweep failed"))); - trackHeartbeatSchedulerWork(questionResponseDeliveries.sweepPending() - .then((result) => { - if (result.scanned > 0) { - logger.info(result, "periodic question-response delivery sweep completed"); - } - }) - .catch((err) => { - logger.error({ err }, "periodic question-response delivery sweep failed"); - })); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { @@ -1974,6 +1965,7 @@ async function startServerWithDatabaseTeardown( await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]); heartbeatSchedulerStopped = true; unsubscribeChatCompletions(); + await deliveryWork.stop(); clearInterval(executionControlInterval); if (heartbeatSchedulerInterval) { clearInterval(heartbeatSchedulerInterval); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 9883c2aa6f6..108ba699448 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -3384,14 +3384,6 @@ export function issueRoutes( ChatChannelService, "prepareFailedChatRunRetry" | "processFailedChatRunRetry" >; - feedbackExportService?: { - flushPendingFeedbackTraces(input?: { - companyId?: string; - traceId?: string; - limit?: number; - now?: Date; - }): Promise; - }; searchService?: CompanySearchService; searchRateLimiter?: CompanySearchRateLimiter; pluginWorkerManager?: PluginWorkerManager; @@ -3647,7 +3639,6 @@ export function issueRoutes( const treeControlSvc = issueTreeControlFactory?.(db) ?? { getActivePauseHoldGate: async () => null, }; - const feedbackExportService = opts?.feedbackExportService; const environmentsSvc = environmentService(db); async function queueTaskWatchdogEvaluation( @@ -19005,21 +18996,8 @@ export function issueRoutes( ); } - if (result.sharingEnabled && result.traceId && feedbackExportService) { - try { - await feedbackExportService.flushPendingFeedbackTraces({ - companyId: issue.companyId, - traceId: result.traceId, - limit: 1, - }); - } catch (err) { - logger.warn( - { err, issueId: issue.id, traceId: result.traceId }, - "failed to flush shared feedback trace immediately", - ); - } - } - + // saveIssueVote has committed the export and notified its worker. + // Return the saved vote without waiting for an upload or backlog. res.status(201).json(result.vote); }, ); diff --git a/server/src/services/chat-completion-delivery.ts b/server/src/services/chat-completion-delivery.ts index 1cbd95b5009..18ddabd2548 100644 --- a/server/src/services/chat-completion-delivery.ts +++ b/server/src/services/chat-completion-delivery.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, asc, eq, inArray, lte, sql } from "drizzle-orm"; import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs, heartbeatRuns, issueComments, issueDocuments, issues, type Db } from "@paperclipai/db"; @@ -31,13 +32,16 @@ export async function recordChatHandoff(tx: Connection, task: Issue, actorRunId: } /** Must run on the same transaction as status projection (including native arbitration). */ -export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue) { - if (before.status === after.status) return; +export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue): Promise { + if (before.status === after.status) return false; await tx.update(deliveries).set({ status: "superseded" }).where(and(eq(deliveries.taskId, after.id), eq(deliveries.companyId, after.companyId), inArray(deliveries.status, [...pending]))); - if (after.status !== "done") return; + if (after.status !== "done") return false; const [handoff] = await tx.select().from(handoffs).where(and(eq(handoffs.taskId, after.id), eq(handoffs.companyId, after.companyId))); - if (handoff) await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + if (!handoff) return false; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatCompletion); + await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + return true; } async function loadAudience(tx: Connection, deliveryId: string) { @@ -243,5 +247,9 @@ export function chatCompletionDeliveryService(db: Db, heartbeat: { wakeup(agentI .orderBy(asc(deliveries.nextAttemptAt)).limit(100); for (const row of due) await deliver(row.id); } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: deliveries.id }).from(deliveries) + .where(inArray(deliveries.status, [...pending])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/connection-intent-delivery.ts b/server/src/services/connection-intent-delivery.ts index 540524bf4ea..c128d309fcd 100644 --- a/server/src/services/connection-intent-delivery.ts +++ b/server/src/services/connection-intent-delivery.ts @@ -167,7 +167,11 @@ export function connectionIntentDeliveryService(db: Db, heartbeat: Pick { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { + async function hasPending() { + return (await db.select({ id: connectionIntentDeliveries.interactionId }).from(connectionIntentDeliveries) + .where(isNull(connectionIntentDeliveries.deliveredAt)).limit(1)).length > 0; + } + return { deliver, hasPending, tryDeliver: async (id: string) => { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { const rows = await db.select().from(connectionIntentDeliveries).where(and(isNull(connectionIntentDeliveries.deliveredAt), lte(connectionIntentDeliveries.nextAttemptAt, new Date()))) .orderBy(asc(connectionIntentDeliveries.nextAttemptAt)).limit(50); let failed = 0; diff --git a/server/src/services/delivery-work-coordinator.ts b/server/src/services/delivery-work-coordinator.ts new file mode 100644 index 00000000000..34a198671ba --- /dev/null +++ b/server/src/services/delivery-work-coordinator.ts @@ -0,0 +1,100 @@ +import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db"; +import { createWorkScheduler } from "./work-scheduler.js"; +import { type DeliveryQueue } from "./delivery-work-notifications.js"; +import { beginIdleTrackedWork } from "./task-admission.js"; + +export function createDeliveryWorkCoordinator(input: { + owner: object; + canRun: () => boolean; + // Idle drain may reconcile admitted writes without dispatching new work. + // Warm standby can disable both operations. + canReconcile?: () => boolean; + onError: (error: unknown, queue: DeliveryQueue) => void; +}) { + const scheduler = createWorkScheduler(); + const workers = new Map void; stop: () => Promise }>(); + let stopped = false; + + return { + // Only these registered tasks are represented; not yet the whole instance. + nextWakeAt: scheduler.nextWakeAt, + register(queue: DeliveryQueue, task: { + retryMs: number; + run: (signal: AbortSignal) => Promise; + hasPending: () => Promise; + }) { + if (stopped) throw new Error("Delivery coordinator is stopped"); + if (workers.has(queue)) throw new Error(`Delivery worker already registered: ${queue}`); + let dirty = false; + let scheduledAt: number | null = null; + let running: Promise | null = null; + let workerStopped = false; + let controller: AbortController | null = null; + let finishIntent: (() => void) | undefined; + function schedule(delay: number) { + if (workerStopped) return; + const at = Date.now() + delay; + if (scheduledAt !== null && scheduledAt <= at) return; + scheduledAt = at; + scheduler.schedule(queue, at, start); + } + function start() { + if (workerStopped || running) return; + scheduler.cancel(queue); + scheduledAt = null; + if (!input.canRun() && !(input.canReconcile?.() ?? false)) { schedule(task.retryMs); return; } + dirty = false; + const finish = beginIdleTrackedWork(); + const attempt = new AbortController(); + controller = attempt; + running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) + .then(() => workerStopped || !input.canRun() ? undefined : task.run(attempt.signal)).then(() => workerStopped ? false : task.hasPending()).then(pending => { + if (pending || databaseWorkPending(input.owner, queue)) schedule(task.retryMs); + else { finishIntent?.(); finishIntent = undefined; } + }).catch(error => { + schedule(task.retryMs); + // Logging must never turn a recoverable sweep failure into an + // unhandled rejection or prevent other queues from running. + try { if (!workerStopped) input.onError(error, queue); } catch { /* Recovery remains scheduled. */ } + }).finally(() => { + running = null; + controller = null; + finish(); + if (dirty) schedule(0); + }); + } + function wake() { + if (workerStopped) return; + dirty = true; + if (!running) schedule(0); + } + const unsubscribe = subscribeDatabaseWork(input.owner, queue, event => { + if (workerStopped) return; + if (databaseWorkPending(input.owner, queue)) { + finishIntent ??= beginIdleTrackedWork(); + // Give the transaction time to settle. Its settlement notification + // replaces this deadline with an immediate wake on the normal path. + if (!running) schedule(task.retryMs); + } + if (event === "settled") wake(); + }); + const worker = { wake, async stop() { + workerStopped = true; + controller?.abort(new Error("Delivery worker stopped")); + unsubscribe(); + scheduler.cancel(queue); + await running; + finishIntent?.(); finishIntent = undefined; + } }; + workers.set(queue, worker); + const ready = Promise.resolve().then(() => { start(); return running; }); + return { ready, wake }; + }, + async stop() { + stopped = true; + scheduler.stop(); + await Promise.all(Array.from(workers.values(), worker => worker.stop())); + workers.clear(); + }, + }; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts new file mode 100644 index 00000000000..14b67085d1d --- /dev/null +++ b/server/src/services/delivery-work-notifications.ts @@ -0,0 +1,22 @@ +import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; + +/** Topics for existing durable queues; register intent before their writes. */ +export const DELIVERY_QUEUES = { + feedback: "feedback-exports", + chatCompletion: "chat-completions", + connection: "connection-continuations", + question: "question-responses", + toolAction: "tool-action-receipts", +} as const; +export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; + +export async function notifyDeliveryWork(transaction: object, queue: DeliveryQueue): Promise { + await signalDatabaseWork(transaction, queue); +} +// Public delivery notification observers only see settled work. The coordinator +// uses the lower-level lifecycle subscription to also fence in-flight writes. +export function subscribeDeliveryWork(owner: object, queue: DeliveryQueue, wake: () => void): () => void { + return subscribeDatabaseWork(owner, queue, event => { + if (event === "settled") wake(); + }); +} diff --git a/server/src/services/feedback-share-client.ts b/server/src/services/feedback-share-client.ts index 7fdf97ca4af..ae5a308829d 100644 --- a/server/src/services/feedback-share-client.ts +++ b/server/src/services/feedback-share-client.ts @@ -2,6 +2,8 @@ import { gzipSync } from "node:zlib"; import type { FeedbackTraceBundle } from "@paperclipai/shared"; import type { Config } from "../config.js"; +export const FEEDBACK_UPLOAD_TIMEOUT_MS = 30_000; + const DEFAULT_FEEDBACK_EXPORT_BACKEND_URL = "https://telemetry.paperclip.ing"; function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Date) { @@ -12,7 +14,7 @@ function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Da } export interface FeedbackTraceShareClient { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; } export function createFeedbackTraceShareClientFromConfig( @@ -23,7 +25,10 @@ export function createFeedbackTraceShareClientFromConfig( const endpoint = new URL("/feedback-traces", baseUrl).toString(); return { - async uploadTraceBundle(bundle) { + async uploadTraceBundle(bundle, shutdownSignal) { + const deadline = AbortSignal.timeout(FEEDBACK_UPLOAD_TIMEOUT_MS); + const signal = shutdownSignal ? AbortSignal.any([shutdownSignal, deadline]) : deadline; + signal.throwIfAborted(); const exportedAt = new Date(); const objectKey = buildFeedbackShareObjectKey(bundle, exportedAt); const requestBody = JSON.stringify({ @@ -33,6 +38,7 @@ export function createFeedbackTraceShareClientFromConfig( }); const response = await fetch(endpoint, { method: "POST", + signal, headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}), @@ -49,6 +55,8 @@ export function createFeedbackTraceShareClientFromConfig( } const payload = await response.json().catch(() => null) as { objectKey?: unknown } | null; + // A timed-out response body is not a successful upload acknowledgement. + signal.throwIfAborted(); return { objectKey: typeof payload?.objectKey === "string" && payload.objectKey.trim().length > 0 ? payload.objectKey diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts index 2bbc0361a3c..1f9ceb78721 100644 --- a/server/src/services/feedback.ts +++ b/server/src/services/feedback.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { readFile, readdir } from "node:fs/promises"; import path from "node:path"; import { and, asc, desc, eq, getTableColumns, gte, isNull, lte, ne, or } from "drizzle-orm"; @@ -111,7 +112,7 @@ const feedbackExportColumns = getTableColumns(feedbackExports); const instructionsSvc = agentInstructionsService(); type FeedbackTraceShareClient = { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; }; type FeedbackServiceOptions = { @@ -1715,6 +1716,15 @@ async function buildFeedbackTraceBundleFromRow( } export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { + // Serialize explicit flushes and the worker's batches so callers cannot + // upload the same trace concurrently. HTTP vote handlers only save and notify. + let exportFlush: Promise = Promise.resolve(); + function serializeExportFlush(operation: () => Promise): Promise { + const result = exportFlush.then(operation, operation); + exportFlush = result.catch(() => undefined); + return result; + } + return { listIssueVotesForUser: async (issueId: string, authorUserId: string) => db @@ -1786,13 +1796,23 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { return row ? buildFeedbackTraceBundleFromRow(db, row) : null; }, + hasPendingFeedbackTraces: async () => { + const pending = options.shareClient + ? or(eq(feedbackExports.status, "pending"), eq(feedbackExports.status, "failed")) + : eq(feedbackExports.status, "pending"); + return (await db.select({ id: feedbackExports.id }).from(feedbackExports) + .where(pending).limit(1)).length > 0; + }, + flushPendingFeedbackTraces: async (input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; - }) => { + signal?: AbortSignal; + }) => serializeExportFlush(async () => { const shareClient = options.shareClient; + input?.signal?.throwIfAborted(); if (!shareClient) { const filters = [eq(feedbackExports.status, "pending")]; if (input?.companyId) { @@ -1861,12 +1881,13 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { let failed = 0; for (const row of rows) { + if (input?.signal?.aborted) break; const attemptAt = input?.now ?? new Date(); attempted += 1; try { const bundle = await buildFeedbackTraceBundleFromRow(db, row); - await shareClient.uploadTraceBundle(bundle); + await shareClient.uploadTraceBundle(bundle, input?.signal); await db .update(feedbackExports) @@ -1900,7 +1921,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { sent, failed, }; - }, + }), saveIssueVote: async (input: { issueId: string; @@ -2071,6 +2092,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { }) .where(eq(feedbackVotes.id, savedVote.id)); + if (sharedWithLabs) await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); const [savedTrace] = await tx .insert(feedbackExports) .values({ diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index d28d6a4ec07..58555ccb48d 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,4 +1,5 @@ import { normalizeEscapedLineBreaks } from "@paperclipai/shared/validators/text"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { activeIssueInteractionCondition, historicalQuestionCondition } from "./issue-question-context.js"; import { currentContinuationOrigins, @@ -2829,6 +2830,7 @@ export function issueThreadInteractionService( .returning(); if (!row) throw interactionAlreadyResolvedError(); if (status === "accepted" || status === "rejected") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); await tx.insert(connectionIntentDeliveries).values({ interactionId, companyId: issue.companyId }).onConflictDoNothing(); } return row; @@ -5026,6 +5028,7 @@ export function issueThreadInteractionService( // This answer updates conversation history only. It must not resume // the completed source run or enqueue new work for the closed task. if (!historicalAnswer) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.question); await tx .insert(issueQuestionResponseDeliveries) .values(questionResponseDeliveryValues(answered)); diff --git a/server/src/services/question-response-delivery.ts b/server/src/services/question-response-delivery.ts index 5db1e051508..dad35c77d7e 100644 --- a/server/src/services/question-response-delivery.ts +++ b/server/src/services/question-response-delivery.ts @@ -1340,5 +1340,9 @@ export function questionResponseDeliveryService( return counts; } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: issueQuestionResponseDeliveries.id }).from(issueQuestionResponseDeliveries) + .where(inArray(issueQuestionResponseDeliveries.status, ["pending", "delivering"])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/tool-action-delivery.ts b/server/src/services/tool-action-delivery.ts index 63f217de9a0..0718ed2d449 100644 --- a/server/src/services/tool-action-delivery.ts +++ b/server/src/services/tool-action-delivery.ts @@ -345,6 +345,12 @@ export function toolActionDeliveryService( } return { deliver, + async hasPending() { + // Include receipts awaiting review/execution: their later terminal write + // must remain recoverable even if its optional fast-path callback fails. + return (await db.select({ id: toolActionDeliveries.actionRequestId }).from(toolActionDeliveries) + .where(isNull(toolActionDeliveries.deliveredAt)).limit(1)).length > 0; + }, async deliverForRun(input: { companyId: string; runId: string }) { // A review can settle before its source run yields. Retry that run's // receipts as soon as execution cleanup finishes; the periodic sweep diff --git a/server/src/services/tool-action-review.ts b/server/src/services/tool-action-review.ts index b429b35ce69..3d0effa9f07 100644 --- a/server/src/services/tool-action-review.ts +++ b/server/src/services/tool-action-review.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { issues, @@ -206,6 +207,7 @@ export async function commitToolActionReview( : { version: 1, outcome: "rejected", reason: input.reason }, }) .where(eq(issueThreadInteractions.id, interaction.id)); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); await tx .insert(toolActionDeliveries) .values({ diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 86d79cd2f9c..4ef4f1092e8 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { composeConnectionInstructions } from "./connection-instructions.js"; import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } from "./connection-permission-errors.js"; import { boundedMcpToolName } from "./mcp-tool-names.js"; @@ -2615,15 +2616,18 @@ export function createToolGatewayService( ); } - await db - .insert(toolActionDeliveries) - .values({ - companyId: input.session.companyId, - actionRequestId: actionRequest.id, - issueId: input.session.issueId, - interactionId: interaction.id, - }) - .onConflictDoNothing(); + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); + await tx + .insert(toolActionDeliveries) + .values({ + companyId: input.session.companyId, + actionRequestId: actionRequest.id, + issueId: input.session.issueId!, + interactionId: interaction.id, + }) + .onConflictDoNothing(); + }); await writeToolCallEvent({ invocationId: input.invocation.id, diff --git a/server/src/services/work-scheduler.ts b/server/src/services/work-scheduler.ts new file mode 100644 index 00000000000..1d2cdadfe36 --- /dev/null +++ b/server/src/services/work-scheduler.ts @@ -0,0 +1,45 @@ +/** In-process deadlines. Owners restore durable tasks at startup. */ +export function createWorkScheduler() { + const tasks = new Map void }>(); + let timer: ReturnType | null = null; + let stopped = false; + const nextWakeAt = () => tasks.size ? Math.min(...Array.from(tasks.values(), task => task.at)) : null; + + function arm() { + if (timer) clearTimeout(timer); + timer = null; + const next = nextWakeAt(); + if (stopped || next === null) return; + timer = setTimeout(() => { + timer = null; + const now = Date.now(); + const due = Array.from(tasks.entries()).filter(([, task]) => task.at <= now); + for (const [key, task] of due) { + // Another due callback can cancel or replace this task. + if (tasks.get(key) !== task) continue; + tasks.delete(key); + try { task.run(); } + catch (error) { process.emitWarning(`Scheduled work failed: ${String(error)}`); } + } + arm(); + }, Math.min(2_147_483_647, Math.max(0, next - Date.now()))); + timer.unref?.(); + } + + return { + nextWakeAt, + schedule(key: string, at: number, run: () => void) { + if (stopped) return; + if (!Number.isFinite(at)) throw new Error("Work deadline must be finite"); + tasks.set(key, { at, run }); + arm(); + }, + cancel(key: string) { tasks.delete(key); arm(); }, + stop() { + stopped = true; + tasks.clear(); + if (timer) clearTimeout(timer); + timer = null; + }, + }; +}